From 48559e1a1dc533cc3658e319f48c300cba967ffa Mon Sep 17 00:00:00 2001 From: Xiaolu Dai Date: Tue, 24 Dec 2024 19:25:58 +0800 Subject: [PATCH] add azure postgresql mi support --- cli/azd/internal/appdetect/appdetect.go | 9 + cli/azd/internal/auth_type.go | 29 ++ cli/azd/internal/repository/app_init.go | 222 ++++++++++++- cli/azd/internal/repository/app_init_test.go | 11 +- cli/azd/internal/repository/infra_confirm.go | 54 ++-- .../internal/repository/infra_confirm_test.go | 14 +- cli/azd/internal/scaffold/bicep_env.go | 185 +++++++++++ cli/azd/internal/scaffold/bicep_env_test.go | 114 +++++++ cli/azd/internal/scaffold/scaffold.go | 8 +- cli/azd/internal/scaffold/scaffold_test.go | 20 +- cli/azd/internal/scaffold/spec.go | 31 +- .../internal/scaffold/spec_service_binding.go | 305 ++++++++++++++++++ .../scaffold/spec_service_binding_test.go | 168 ++++++++++ cli/azd/pkg/project/importer.go | 4 +- cli/azd/pkg/project/importer_test.go | 9 + cli/azd/pkg/project/resources.go | 18 +- cli/azd/pkg/project/scaffold_gen.go | 192 ++++++++--- cli/azd/pkg/project/service_config.go | 2 + .../scaffold/templates/resources.bicept | 162 ++++++---- cli/azd/test/functional/init_test.go | 1 + 20 files changed, 1372 insertions(+), 186 deletions(-) create mode 100644 cli/azd/internal/auth_type.go create mode 100644 cli/azd/internal/scaffold/bicep_env.go create mode 100644 cli/azd/internal/scaffold/bicep_env_test.go create mode 100644 cli/azd/internal/scaffold/spec_service_binding.go create mode 100644 cli/azd/internal/scaffold/spec_service_binding_test.go diff --git a/cli/azd/internal/appdetect/appdetect.go b/cli/azd/internal/appdetect/appdetect.go index e6103d3cbd7..e86a78187d3 100644 --- a/cli/azd/internal/appdetect/appdetect.go +++ b/cli/azd/internal/appdetect/appdetect.go @@ -131,6 +131,12 @@ func (db DatabaseDep) Display() string { return "" } +type Metadata struct { + ApplicationName string + DatabaseNameInPropertySpringDatasourceUrl map[DatabaseDep]string + ContainsDependencySpringCloudAzureStarterJdbcPostgresql bool +} + type Project struct { // The language associated with the project. Language Language @@ -141,6 +147,9 @@ type Project struct { // Experimental: Database dependencies inferred through heuristics while scanning dependencies in the project. DatabaseDeps []DatabaseDep + // Experimental: Metadata inferred through heuristics while scanning the project. + Metadata Metadata + // The path to the project directory. Path string diff --git a/cli/azd/internal/auth_type.go b/cli/azd/internal/auth_type.go new file mode 100644 index 00000000000..0399d327079 --- /dev/null +++ b/cli/azd/internal/auth_type.go @@ -0,0 +1,29 @@ +package internal + +// AuthType defines different authentication types. +type AuthType string + +const ( + AuthTypeUnspecified AuthType = "unspecified" + // Username and password, or key based authentication + AuthTypePassword AuthType = "password" + // Connection string authentication + AuthTypeConnectionString AuthType = "connectionString" + // Microsoft EntraID token credential + AuthTypeUserAssignedManagedIdentity AuthType = "userAssignedManagedIdentity" +) + +func GetAuthTypeDescription(authType AuthType) string { + switch authType { + case AuthTypeUnspecified: + return "Unspecified" + case AuthTypePassword: + return "Username and password" + case AuthTypeConnectionString: + return "Connection string" + case AuthTypeUserAssignedManagedIdentity: + return "User assigned managed identity" + default: + return "Unspecified" + } +} diff --git a/cli/azd/internal/repository/app_init.go b/cli/azd/internal/repository/app_init.go index cb211bc14b1..9586bcc9571 100644 --- a/cli/azd/internal/repository/app_init.go +++ b/cli/azd/internal/repository/app_init.go @@ -6,6 +6,7 @@ import ( "maps" "os" "path/filepath" + "regexp" "slices" "strings" "time" @@ -424,31 +425,56 @@ func (i *Initializer) prjConfigFromDetect( continue } - var dbType project.ResourceType - switch database { - case appdetect.DbMongo: - dbType = project.ResourceTypeDbMongo - case appdetect.DbPostgres: - dbType = project.ResourceTypeDbPostgres - } - - db := project.ResourceConfig{ - Type: dbType, + var err error + databaseName, err := getDatabaseName(database, &detect, i.console, ctx) + if err != nil { + return config, err } - for { - dbName, err := promptDbName(i.console, ctx, database) + var authType = internal.AuthTypeUnspecified + if database == appdetect.DbPostgres { + var err error + authType, err = chooseAuthTypeByPrompt( + database.Display(), + []internal.AuthType{internal.AuthTypeUserAssignedManagedIdentity, internal.AuthTypePassword}, + ctx, + i.console) if err != nil { return config, err } - - if dbName == "" { - i.console.Message(ctx, "Database name is required.") + continueProvision, err := checkPasswordlessConfigurationAndContinueProvision(database, authType, &detect, + i.console, ctx) + if err != nil { + return config, err + } + if !continueProvision { continue } + } - db.Name = dbName - break + if database == appdetect.DbPostgres { + postgres := project.ResourceConfig{ + Type: project.ResourceTypeDbPostgres, + Name: "postgresql", + Props: project.PostgresProps{ + DatabaseName: databaseName, + AuthType: authType, + }, + } + config.Resources[postgres.Name] = &postgres + dbNames[database] = postgres.Name + continue + } + + var dbType project.ResourceType + switch database { + case appdetect.DbMongo: + dbType = project.ResourceTypeDbMongo + } + + db := project.ResourceConfig{ + Type: dbType, + Name: databaseName, } config.Resources[db.Name] = &db @@ -578,3 +604,165 @@ func ServiceFromDetect( return svc, nil } + +func chooseAuthTypeByPrompt( + name string, + authOptions []internal.AuthType, + ctx context.Context, + console input.Console) (internal.AuthType, error) { + var options []string + for _, option := range authOptions { + options = append(options, internal.GetAuthTypeDescription(option)) + } + selection, err := console.Select(ctx, input.ConsoleOptions{ + Message: "Choose auth type for " + name + ":", + Options: options, + }) + if err != nil { + return internal.AuthTypeUnspecified, err + } + return authOptions[selection], nil +} + +func checkPasswordlessConfigurationAndContinueProvision(database appdetect.DatabaseDep, authType internal.AuthType, + detect *detectConfirm, console input.Console, ctx context.Context) (bool, error) { + if authType != internal.AuthTypeUserAssignedManagedIdentity { + return true, nil + } + for i, prj := range detect.Services { + if lackedDep := lackedAzureStarterJdbcDependency(prj, database); lackedDep != "" { + message := fmt.Sprintf("\nError!\n"+ + "You selected '%s' as auth type for '%s'.\n"+ + "For this auth type, this dependency is required:\n"+ + "%s\n"+ + "But this dependency is not found in your project:\n"+ + "%s", + internal.AuthTypeUserAssignedManagedIdentity, database, lackedDep, prj.Path) + continueOption, err := console.Select(ctx, input.ConsoleOptions{ + Message: fmt.Sprintf("%s\nSelect an option:", message), + Options: []string{ + "Exit azd and fix problem manually", + fmt.Sprintf("Continue azd and use %s in this project: %s", database.Display(), prj.Path), + fmt.Sprintf("Continue azd and not use %s in this project: %s", database.Display(), prj.Path), + }, + }) + if err != nil { + return false, err + } + + switch continueOption { + case 0: + os.Exit(0) + case 1: + continue + case 2: + // remove related database usage + var result []appdetect.DatabaseDep + for _, db := range prj.DatabaseDeps { + if db != database { + result = append(result, db) + } + } + prj.DatabaseDeps = result + detect.Services[i] = prj + // delete database if no other service used + dbUsed := false + for _, svc := range detect.Services { + for _, db := range svc.DatabaseDeps { + if db == database { + dbUsed = true + break + } + } + if dbUsed { + break + } + } + if !dbUsed { + console.Message(ctx, fmt.Sprintf( + "Deleting database %s due to no service used", database.Display())) + delete(detect.Databases, database) + return false, nil + } + } + } + } + return true, nil +} + +func lackedAzureStarterJdbcDependency(project appdetect.Project, database appdetect.DatabaseDep) string { + if project.Language != appdetect.Java { + return "" + } + + useDatabase := false + for _, db := range project.DatabaseDeps { + if db == database { + useDatabase = true + break + } + } + if !useDatabase { + return "" + } + if database == appdetect.DbPostgres && !project.Metadata.ContainsDependencySpringCloudAzureStarterJdbcPostgresql { + return "\n" + + " com.azure.spring\n" + + " spring-cloud-azure-starter-jdbc-postgresql\n" + + " xxx\n" + + "" + } + return "" +} + +func getDatabaseName(database appdetect.DatabaseDep, detect *detectConfirm, + console input.Console, ctx context.Context) (string, error) { + dbName := getDatabaseNameFromProjectMetadata(detect, database) + if dbName != "" { + return dbName, nil + } + for { + dbName, err := console.Prompt(ctx, input.ConsoleOptions{ + Message: fmt.Sprintf("Input the databaseName for %s "+ + "(Not databaseServerName. This url can explain the difference: "+ + "'jdbc:mysql://databaseServerName:3306/databaseName'):", database.Display()), + Help: "Hint: App database name\n\n" + + "Name of the database that the app connects to. " + + "This database will be created after running azd provision or azd up.\n" + + "You may be able to skip this step by hitting enter, in which case the database will not be created.", + }) + if err != nil { + return "", err + } + if isValidDatabaseName(dbName) { + return dbName, nil + } else { + console.Message(ctx, "Invalid database name. Please choose another name.") + } + } +} + +func getDatabaseNameFromProjectMetadata(detect *detectConfirm, database appdetect.DatabaseDep) string { + result := "" + for _, service := range detect.Services { + // todo this should not be here, it should be part of the app detect + name := service.Metadata.DatabaseNameInPropertySpringDatasourceUrl[database] + if name != "" { + if result == "" { + result = name + } else { + // different project configured different db name, not use any of them. + return "" + } + } + } + return result +} + +func isValidDatabaseName(name string) bool { + if len(name) < 3 || len(name) > 63 { + return false + } + re := regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`) + return re.MatchString(name) +} diff --git a/cli/azd/internal/repository/app_init_test.go b/cli/azd/internal/repository/app_init_test.go index 6a0b7a7dac6..f7a7495650b 100644 --- a/cli/azd/internal/repository/app_init_test.go +++ b/cli/azd/internal/repository/app_init_test.go @@ -216,6 +216,7 @@ func TestInitializer_prjConfigFromDetect(t *testing.T) { "my$special$db", "n", "postgres", // fill in db name + "Username and password", }, want: project.ProjectConfig{ Services: map[string]*project.ServiceConfig{ @@ -240,14 +241,18 @@ func TestInitializer_prjConfigFromDetect(t *testing.T) { Type: project.ResourceTypeDbMongo, Name: "mongodb", }, - "postgres": { + "postgresql": { Type: project.ResourceTypeDbPostgres, - Name: "postgres", + Name: "postgresql", + Props: project.PostgresProps{ + AuthType: internal.AuthTypePassword, + DatabaseName: "postgres", + }, }, "py": { Type: project.ResourceTypeHostContainerApp, Name: "py", - Uses: []string{"postgres", "mongodb", "redis"}, + Uses: []string{"postgresql", "mongodb", "redis"}, Props: project.ContainerAppProps{ Port: 80, }, diff --git a/cli/azd/internal/repository/infra_confirm.go b/cli/azd/internal/repository/infra_confirm.go index ee5a50f716b..c16d14ecd44 100644 --- a/cli/azd/internal/repository/infra_confirm.go +++ b/cli/azd/internal/repository/infra_confirm.go @@ -3,6 +3,7 @@ package repository import ( "context" "fmt" + "github.com/azure/azure-dev/cli/azd/internal" "path/filepath" "regexp" "strconv" @@ -31,6 +32,34 @@ func (i *Initializer) infraSpecFromDetect( continue } + if database == appdetect.DbPostgres { + dbName, err := getDatabaseName(database, &detect, i.console, ctx) + if err != nil { + return scaffold.InfraSpec{}, err + } + authType, err := chooseAuthTypeByPrompt( + database.Display(), + []internal.AuthType{internal.AuthTypeUserAssignedManagedIdentity, internal.AuthTypePassword}, + ctx, + i.console) + if err != nil { + return scaffold.InfraSpec{}, err + } + continueProvision, err := checkPasswordlessConfigurationAndContinueProvision(database, + authType, &detect, i.console, ctx) + if err != nil { + return scaffold.InfraSpec{}, err + } + if !continueProvision { + continue + } + spec.DbPostgres = &scaffold.DatabasePostgres{ + DatabaseName: dbName, + AuthType: authType, + } + continue + } + dbPrompt: for { dbName, err := promptDbName(i.console, ctx, database) @@ -44,15 +73,6 @@ func (i *Initializer) infraSpecFromDetect( DatabaseName: dbName, } break dbPrompt - case appdetect.DbPostgres: - if dbName == "" { - i.console.Message(ctx, "Database name is required.") - continue - } - - spec.DbPostgres = &scaffold.DatabasePostgres{ - DatabaseName: dbName, - } } break dbPrompt } @@ -85,19 +105,17 @@ func (i *Initializer) infraSpecFromDetect( switch db { case appdetect.DbMongo: - serviceSpec.DbCosmosMongo = &scaffold.DatabaseReference{ - DatabaseName: spec.DbCosmosMongo.DatabaseName, - } + err = scaffold.BindToMongoDb(&serviceSpec, spec.DbCosmosMongo) case appdetect.DbPostgres: - serviceSpec.DbPostgres = &scaffold.DatabaseReference{ - DatabaseName: spec.DbPostgres.DatabaseName, - } + err = scaffold.BindToPostgres(&serviceSpec, spec.DbPostgres) case appdetect.DbRedis: - serviceSpec.DbRedis = &scaffold.DatabaseReference{ - DatabaseName: "redis", - } + err = scaffold.BindToRedis(&serviceSpec, spec.DbRedis) } } + + if err != nil { + return scaffold.InfraSpec{}, err + } spec.Services = append(spec.Services, serviceSpec) } diff --git a/cli/azd/internal/repository/infra_confirm_test.go b/cli/azd/internal/repository/infra_confirm_test.go index 7ccfdfeab25..7fcf6c13f1d 100644 --- a/cli/azd/internal/repository/infra_confirm_test.go +++ b/cli/azd/internal/repository/infra_confirm_test.go @@ -14,6 +14,11 @@ import ( ) func TestInitializer_infraSpecFromDetect(t *testing.T) { + dbPostgres := &scaffold.DatabasePostgres{ + DatabaseName: "myappdb", + AuthType: "userAssignedManagedIdentity", + } + envs, _ := scaffold.GetServiceBindingEnvsForPostgres(*dbPostgres) tests := []struct { name string detect detectConfirm @@ -166,11 +171,13 @@ func TestInitializer_infraSpecFromDetect(t *testing.T) { "n", "my$special$db", "n", - "myappdb", // fill in db name + "myappdb", // fill in db name + "User assigned managed identity", // confirm db authentication }, want: scaffold.InfraSpec{ DbPostgres: &scaffold.DatabasePostgres{ DatabaseName: "myappdb", + AuthType: "userAssignedManagedIdentity", }, Services: []scaffold.ServiceSpec{ { @@ -183,9 +190,8 @@ func TestInitializer_infraSpecFromDetect(t *testing.T) { }, }, }, - DbPostgres: &scaffold.DatabaseReference{ - DatabaseName: "myappdb", - }, + DbPostgres: dbPostgres, + Envs: envs, }, { Name: "js", diff --git a/cli/azd/internal/scaffold/bicep_env.go b/cli/azd/internal/scaffold/bicep_env.go new file mode 100644 index 00000000000..f8dd49c27fb --- /dev/null +++ b/cli/azd/internal/scaffold/bicep_env.go @@ -0,0 +1,185 @@ +package scaffold + +import ( + "fmt" + "strings" +) + +func ToBicepEnv(env Env) BicepEnv { + if isServiceBindingEnvValue(env.Value) { + serviceType, infoType := toServiceTypeAndServiceBindingInfoType(env.Value) + value, ok := bicepEnv[serviceType][infoType] + if !ok { + panic(unsupportedType(env)) + } + if isSecret(infoType) { + if isKeyVaultSecret(value) { + return BicepEnv{ + BicepEnvType: BicepEnvTypeKeyVaultSecret, + Name: env.Name, + SecretName: secretName(env), + SecretValue: unwrapKeyVaultSecretValue(value), + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypeSecret, + Name: env.Name, + SecretName: secretName(env), + SecretValue: value, + } + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: env.Name, + PlainTextValue: value, + } + } + } else { + return BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: env.Name, + PlainTextValue: toBicepEnvPlainTextValue(env.Value), + } + } +} + +// inputStringExample -> 'inputStringExample' +func addQuotation(input string) string { + return fmt.Sprintf("'%s'", input) +} + +// 'inputStringExample' -> 'inputStringExample' +// '${inputSingleVariableExample}' -> inputSingleVariableExample +// '${HOST}:${PORT}' -> '${HOST}:${PORT}' +func removeQuotationIfItIsASingleVariable(input string) string { + prefix := "'${" + suffix := "}'" + if strings.HasPrefix(input, prefix) && strings.HasSuffix(input, suffix) { + prefixTrimmed := strings.TrimPrefix(input, prefix) + trimmed := strings.TrimSuffix(prefixTrimmed, suffix) + if !strings.ContainsAny(trimmed, "}") { + return trimmed + } else { + return input + } + } else { + return input + } +} + +// The BicepEnv.PlainTextValue is handled as variable by default. +// If the value is string, it should contain ('). +// Here are some examples of input and output: +// inputStringExample -> 'inputStringExample' +// ${inputSingleVariableExample} -> inputSingleVariableExample +// ${HOST}:${PORT} -> '${HOST}:${PORT}' +func toBicepEnvPlainTextValue(input string) string { + return removeQuotationIfItIsASingleVariable(addQuotation(input)) +} + +// BicepEnv +// +// For Name and SecretName, they are handled as string by default. +// Which means quotation will be added before they are used in bicep file, because they are always string value. +// +// For PlainTextValue and SecretValue, they are handled as variable by default. +// When they are string value, quotation should be contained by themselves. +// Set variable as default is mainly to avoid this problem: +// https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/linter-rule-simplify-interpolation +type BicepEnv struct { + BicepEnvType BicepEnvType + Name string + PlainTextValue string + SecretName string + SecretValue string +} + +type BicepEnvType string + +const ( + BicepEnvTypePlainText BicepEnvType = "plainText" + BicepEnvTypeSecret BicepEnvType = "secret" + BicepEnvTypeKeyVaultSecret BicepEnvType = "keyVaultSecret" +) + +// Note: The value is handled as variable. +// If the value is string, it should contain quotation inside itself. +var bicepEnv = map[ServiceType]map[ServiceBindingInfoType]string{ + ServiceTypeDbPostgres: { + ServiceBindingInfoTypeHost: "postgreServer.outputs.fqdn", + ServiceBindingInfoTypePort: "'5432'", + ServiceBindingInfoTypeDatabaseName: "postgreSqlDatabaseName", + ServiceBindingInfoTypeUsername: "postgreSqlDatabaseUser", + ServiceBindingInfoTypePassword: "postgreSqlDatabasePassword", + ServiceBindingInfoTypeUrl: "'postgresql://${postgreSqlDatabaseUser}:${postgreSqlDatabasePassword}@" + + "${postgreServer.outputs.fqdn}:5432/${postgreSqlDatabaseName}'", + ServiceBindingInfoTypeJdbcUrl: "'jdbc:postgresql://${postgreServer.outputs.fqdn}:5432/" + + "${postgreSqlDatabaseName}'", + }, + ServiceTypeDbRedis: { + ServiceBindingInfoTypeHost: "redis.outputs.hostName", + ServiceBindingInfoTypePort: "string(redis.outputs.sslPort)", + ServiceBindingInfoTypeEndpoint: "'${redis.outputs.hostName}:${redis.outputs.sslPort}'", + ServiceBindingInfoTypePassword: wrapToKeyVaultSecretValue("redisConn.outputs.keyVaultUrlForPass"), + ServiceBindingInfoTypeUrl: wrapToKeyVaultSecretValue("redisConn.outputs.keyVaultUrlForUrl"), + }, + ServiceTypeDbMongo: { + ServiceBindingInfoTypeDatabaseName: "mongoDatabaseName", + ServiceBindingInfoTypeUrl: wrapToKeyVaultSecretValue( + "cosmos.outputs.exportedSecrets['MONGODB-URL'].secretUri", + ), + }, + ServiceTypeOpenAiModel: { + ServiceBindingInfoTypeEndpoint: "account.outputs.endpoint", + }, + ServiceTypeHostContainerApp: { + ServiceBindingInfoTypeHost: "https://{{BackendName}}.${containerAppsEnvironment.outputs.defaultDomain}", + }, +} + +func GetContainerAppHost(name string) string { + return strings.ReplaceAll( + bicepEnv[ServiceTypeHostContainerApp][ServiceBindingInfoTypeHost], + "{{BackendName}}", + name, + ) +} + +func unsupportedType(env Env) string { + return fmt.Sprintf( + "unsupported connection info type for resource type. value = %s", env.Value, + ) +} + +func PlaceHolderForServiceIdentityClientId() string { + return "__PlaceHolderForServiceIdentityClientId" +} + +func isSecret(info ServiceBindingInfoType) bool { + return info == ServiceBindingInfoTypePassword || info == ServiceBindingInfoTypeUrl || + info == ServiceBindingInfoTypeConnectionString +} + +func secretName(env Env) string { + resourceType, resourceInfoType := toServiceTypeAndServiceBindingInfoType(env.Value) + name := fmt.Sprintf("%s-%s", resourceType, resourceInfoType) + lowerCaseName := strings.ToLower(name) + noDotName := strings.Replace(lowerCaseName, ".", "-", -1) + noUnderscoreName := strings.Replace(noDotName, "_", "-", -1) + return noUnderscoreName +} + +var keyVaultSecretPrefix = "keyvault:" + +func isKeyVaultSecret(value string) bool { + return strings.HasPrefix(value, keyVaultSecretPrefix) +} + +func wrapToKeyVaultSecretValue(value string) string { + return fmt.Sprintf("%s%s", keyVaultSecretPrefix, value) +} + +func unwrapKeyVaultSecretValue(value string) string { + return strings.TrimPrefix(value, keyVaultSecretPrefix) +} diff --git a/cli/azd/internal/scaffold/bicep_env_test.go b/cli/azd/internal/scaffold/bicep_env_test.go new file mode 100644 index 00000000000..bafefc99e64 --- /dev/null +++ b/cli/azd/internal/scaffold/bicep_env_test.go @@ -0,0 +1,114 @@ +package scaffold + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToBicepEnv(t *testing.T) { + tests := []struct { + name string + in Env + want BicepEnv + }{ + { + name: "Plain text", + in: Env{ + Name: "enable-customer-related-feature", + Value: "true", + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "enable-customer-related-feature", + PlainTextValue: "'true'", // Note: Quotation add automatically + }, + }, + { + name: "Plain text from EnvTypeResourceConnectionPlainText", + in: Env{ + Name: "spring.jms.servicebus.pricing-tier", + Value: "premium", + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "spring.jms.servicebus.pricing-tier", + PlainTextValue: "'premium'", // Note: Quotation add automatically + }, + }, + { + name: "Plain text from EnvTypeResourceConnectionResourceInfo", + in: Env{ + Name: "POSTGRES_PORT", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePort), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypePlainText, + Name: "POSTGRES_PORT", + PlainTextValue: "'5432'", + }, + }, + { + name: "Secret", + in: Env{ + Name: "POSTGRES_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypeSecret, + Name: "POSTGRES_PASSWORD", + SecretName: "db-postgres-password", + SecretValue: "postgreSqlDatabasePassword", + }, + }, + { + name: "KeuVault Secret", + in: Env{ + Name: "REDIS_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbRedis, ServiceBindingInfoTypePassword), + }, + want: BicepEnv{ + BicepEnvType: BicepEnvTypeKeyVaultSecret, + Name: "REDIS_PASSWORD", + SecretName: "db-redis-password", + SecretValue: "redisConn.outputs.keyVaultUrlForPass", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ToBicepEnv(tt.in) + assert.Equal(t, tt.want, actual) + }) + } +} + +func TestToBicepEnvPlainTextValue(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "string", + in: "inputStringExample", + want: "'inputStringExample'", + }, + { + name: "single variable", + in: "${inputSingleVariableExample}", + want: "inputSingleVariableExample", + }, + { + name: "multiple variable", + in: "${HOST}:${PORT}", + want: "'${HOST}:${PORT}'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := toBicepEnvPlainTextValue(tt.in) + assert.Equal(t, tt.want, actual) + }) + } +} diff --git a/cli/azd/internal/scaffold/scaffold.go b/cli/azd/internal/scaffold/scaffold.go index f9ce4752ea9..6f56bc64c5f 100644 --- a/cli/azd/internal/scaffold/scaffold.go +++ b/cli/azd/internal/scaffold/scaffold.go @@ -30,6 +30,8 @@ func Load() (*template.Template, error) { "lower": strings.ToLower, "alphaSnakeUpper": AlphaSnakeUpper, "formatParam": FormatParameter, + "hasPrefix": strings.HasPrefix, + "toBicepEnv": ToBicepEnv, } t, err := template.New("templates"). @@ -201,12 +203,12 @@ func executeToFS(targetFS *memfs.FS, tmpl *template.Template, name string, path } func preExecExpand(spec *InfraSpec) { - // postgres requires specific password seeding parameters + // postgres and mysql requires specific password seeding parameters if spec.DbPostgres != nil { spec.Parameters = append(spec.Parameters, Parameter{ - Name: "databasePassword", - Value: "$(secretOrRandomPassword ${AZURE_KEY_VAULT_NAME} databasePassword)", + Name: "postgreSqlDatabasePassword", + Value: "$(secretOrRandomPassword ${AZURE_KEY_VAULT_NAME} postgreSqlDatabasePassword)", Type: "string", Secret: true, }) diff --git a/cli/azd/internal/scaffold/scaffold_test.go b/cli/azd/internal/scaffold/scaffold_test.go index 238043c3673..d5a7dc212fb 100644 --- a/cli/azd/internal/scaffold/scaffold_test.go +++ b/cli/azd/internal/scaffold/scaffold_test.go @@ -98,13 +98,11 @@ func TestExecInfra(t *testing.T) { }, }, }, - DbCosmosMongo: &DatabaseReference{ + DbCosmosMongo: &DatabaseCosmosMongo{ DatabaseName: "appdb", }, - DbRedis: &DatabaseReference{ - DatabaseName: "redis", - }, - DbPostgres: &DatabaseReference{ + DbRedis: &DatabaseRedis{}, + DbPostgres: &DatabasePostgres{ DatabaseName: "appdb", }, }, @@ -133,7 +131,7 @@ func TestExecInfra(t *testing.T) { { Name: "api", Port: 3100, - DbPostgres: &DatabaseReference{ + DbPostgres: &DatabasePostgres{ DatabaseName: "appdb", }, }, @@ -150,7 +148,7 @@ func TestExecInfra(t *testing.T) { { Name: "api", Port: 3100, - DbCosmosMongo: &DatabaseReference{ + DbCosmosMongo: &DatabaseCosmosMongo{ DatabaseName: "appdb", }, }, @@ -163,11 +161,9 @@ func TestExecInfra(t *testing.T) { DbRedis: &DatabaseRedis{}, Services: []ServiceSpec{ { - Name: "api", - Port: 3100, - DbRedis: &DatabaseReference{ - DatabaseName: "redis", - }, + Name: "api", + Port: 3100, + DbRedis: &DatabaseRedis{}, }, }, }, diff --git a/cli/azd/internal/scaffold/spec.go b/cli/azd/internal/scaffold/spec.go index 763b83c322e..2997479bcc7 100644 --- a/cli/azd/internal/scaffold/spec.go +++ b/cli/azd/internal/scaffold/spec.go @@ -2,6 +2,7 @@ package scaffold import ( "fmt" + "github.com/azure/azure-dev/cli/azd/internal" "strings" ) @@ -28,6 +29,7 @@ type Parameter struct { type DatabasePostgres struct { DatabaseUser string DatabaseName string + AuthType internal.AuthType } type DatabaseCosmosMongo struct { @@ -55,7 +57,7 @@ type ServiceSpec struct { Name string Port int - Env map[string]string + Envs []Env // Front-end properties. Frontend *Frontend @@ -64,14 +66,19 @@ type ServiceSpec struct { Backend *Backend // Connection to a database - DbPostgres *DatabaseReference - DbCosmosMongo *DatabaseReference - DbRedis *DatabaseReference + DbPostgres *DatabasePostgres + DbCosmosMongo *DatabaseCosmosMongo + DbRedis *DatabaseRedis // AI model connections AIModels []AIModelReference } +type Env struct { + Name string + Value string +} + type Frontend struct { Backends []ServiceReference } @@ -140,3 +147,19 @@ func serviceDefPlaceholder(serviceName string) Parameter { Secret: true, } } + +func AddNewEnvironmentVariable(serviceSpec *ServiceSpec, name string, value string) error { + merged, err := mergeEnvWithDuplicationCheck(serviceSpec.Envs, + []Env{ + { + Name: name, + Value: value, + }, + }, + ) + if err != nil { + return err + } + serviceSpec.Envs = merged + return nil +} diff --git a/cli/azd/internal/scaffold/spec_service_binding.go b/cli/azd/internal/scaffold/spec_service_binding.go new file mode 100644 index 00000000000..dafd8e5103a --- /dev/null +++ b/cli/azd/internal/scaffold/spec_service_binding.go @@ -0,0 +1,305 @@ +package scaffold + +import ( + "fmt" + "strings" + + "github.com/azure/azure-dev/cli/azd/internal" +) + +// todo merge ServiceType and project.ResourceType +// Not use project.ResourceType because it will cause cycle import. +// Not merge it in current PR to avoid conflict with upstream main branch. +// Solution proposal: define a ServiceType in lower level that can be used both in scaffold and project package. + +type ServiceType string + +const ( + ServiceTypeDbRedis ServiceType = "db.redis" + ServiceTypeDbPostgres ServiceType = "db.postgres" + ServiceTypeDbMongo ServiceType = "db.mongo" + ServiceTypeHostContainerApp ServiceType = "host.containerapp" + ServiceTypeOpenAiModel ServiceType = "ai.openai.model" +) + +type ServiceBindingInfoType string + +const ( + ServiceBindingInfoTypeHost ServiceBindingInfoType = "host" + ServiceBindingInfoTypePort ServiceBindingInfoType = "port" + ServiceBindingInfoTypeEndpoint ServiceBindingInfoType = "endpoint" + ServiceBindingInfoTypeDatabaseName ServiceBindingInfoType = "databaseName" + ServiceBindingInfoTypeUsername ServiceBindingInfoType = "username" + ServiceBindingInfoTypePassword ServiceBindingInfoType = "password" + ServiceBindingInfoTypeUrl ServiceBindingInfoType = "url" + ServiceBindingInfoTypeJdbcUrl ServiceBindingInfoType = "jdbcUrl" + ServiceBindingInfoTypeConnectionString ServiceBindingInfoType = "connectionString" +) + +var serviceBindingEnvValuePrefix = "$service.binding" + +func isServiceBindingEnvValue(env string) bool { + if !strings.HasPrefix(env, serviceBindingEnvValuePrefix) { + return false + } + a := strings.Split(env, ":") + if len(a) != 3 { + return false + } + return a[0] != "" && a[1] != "" && a[2] != "" +} + +func ToServiceBindingEnvValue(resourceType ServiceType, resourceInfoType ServiceBindingInfoType) string { + return fmt.Sprintf("%s:%s:%s", serviceBindingEnvValuePrefix, resourceType, resourceInfoType) +} + +func toServiceTypeAndServiceBindingInfoType(resourceConnectionEnv string) ( + serviceType ServiceType, infoType ServiceBindingInfoType) { + if !isServiceBindingEnvValue(resourceConnectionEnv) { + return "", "" + } + a := strings.Split(resourceConnectionEnv, ":") + return ServiceType(a[1]), ServiceBindingInfoType(a[2]) +} + +func BindToMongoDb(serviceSpec *ServiceSpec, mongo *DatabaseCosmosMongo) error { + serviceSpec.DbCosmosMongo = mongo + envs := GetServiceBindingEnvsForMongo() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToPostgres(serviceSpec *ServiceSpec, postgres *DatabasePostgres) error { + serviceSpec.DbPostgres = postgres + envs, err := GetServiceBindingEnvsForPostgres(*postgres) + if err != nil { + return err + } + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToRedis(serviceSpec *ServiceSpec, redis *DatabaseRedis) error { + serviceSpec.DbRedis = redis + envs := GetServiceBindingEnvsForRedis() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +func BindToAIModels(serviceSpec *ServiceSpec, model string) error { + serviceSpec.AIModels = append(serviceSpec.AIModels, AIModelReference{Name: model}) + envs := GetServiceBindingEnvsForAIModel() + var err error + serviceSpec.Envs, err = mergeEnvWithDuplicationCheck(serviceSpec.Envs, envs) + if err != nil { + return err + } + return nil +} + +// BindToContainerApp a call b +// todo: +// 1. Add field in ServiceSpec to identify b's app type like Eureka server and Config server. +// 2. Create GetServiceBindingEnvsForContainerApp +// 3. Merge GetServiceBindingEnvsForEurekaServer and GetServiceBindingEnvsForConfigServer into +// GetServiceBindingEnvsForContainerApp. +// 4. Delete printHintsAboutUseHostContainerApp use GetServiceBindingEnvsForContainerApp instead +func BindToContainerApp(a *ServiceSpec, b *ServiceSpec) { + if a.Frontend == nil { + a.Frontend = &Frontend{} + } + a.Frontend.Backends = append(a.Frontend.Backends, ServiceReference{Name: b.Name}) + if b.Backend == nil { + b.Backend = &Backend{} + } + b.Backend.Frontends = append(b.Backend.Frontends, ServiceReference{Name: b.Name}) +} + +func GetServiceBindingEnvsForMongo() []Env { + return []Env{ + { + Name: "MONGODB_URL", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.data.mongodb.uri", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.data.mongodb.database", + Value: ToServiceBindingEnvValue(ServiceTypeDbMongo, ServiceBindingInfoTypeDatabaseName), + }, + } +} + +func GetServiceBindingEnvsForPostgres(postgres DatabasePostgres) ([]Env, error) { + switch postgres.AuthType { + case internal.AuthTypePassword: + return []Env{ + { + Name: "POSTGRES_USERNAME", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "POSTGRES_PASSWORD", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + { + Name: "POSTGRES_HOST", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeHost), + }, + { + Name: "POSTGRES_DATABASE", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeDatabaseName), + }, + { + Name: "POSTGRES_PORT", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePort), + }, + { + Name: "POSTGRES_URL", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUrl), + }, + { + Name: "spring.datasource.url", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeJdbcUrl), + }, + { + Name: "spring.datasource.username", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "spring.datasource.password", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePassword), + }, + }, nil + case internal.AuthTypeUserAssignedManagedIdentity: + return []Env{ + { + Name: "POSTGRES_USERNAME", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "POSTGRES_HOST", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeHost), + }, + { + Name: "POSTGRES_DATABASE", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeDatabaseName), + }, + { + Name: "POSTGRES_PORT", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypePort), + }, + { + Name: "spring.datasource.url", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeJdbcUrl), + }, + { + Name: "spring.datasource.username", + Value: ToServiceBindingEnvValue(ServiceTypeDbPostgres, ServiceBindingInfoTypeUsername), + }, + { + Name: "spring.datasource.password", + Value: "", + }, + { + Name: "spring.datasource.azure.passwordless-enabled", + Value: "true", + }, + { + Name: "spring.cloud.azure.credential.client-id", + Value: PlaceHolderForServiceIdentityClientId(), + }, + { + Name: "spring.cloud.azure.credential.managed-identity-enabled", + Value: "true", + }, + }, nil + default: + return []Env{}, unsupportedAuthTypeError(ServiceTypeDbPostgres, postgres.AuthType) + } +} + +func GetServiceBindingEnvsForRedis() []Env { + return []Env{ + { + Name: "REDIS_HOST", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeHost), + }, + { + Name: "REDIS_PORT", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypePort), + }, + { + Name: "REDIS_ENDPOINT", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeEndpoint), + }, + { + Name: "REDIS_URL", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeUrl), + }, + { + Name: "REDIS_PASSWORD", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypePassword), + }, + { + Name: "spring.data.redis.url", + Value: ToServiceBindingEnvValue( + ServiceTypeDbRedis, ServiceBindingInfoTypeUrl), + }, + } +} + +func GetServiceBindingEnvsForAIModel() []Env { + return []Env{ + { + Name: "AZURE_OPENAI_ENDPOINT", + Value: ToServiceBindingEnvValue(ServiceTypeOpenAiModel, ServiceBindingInfoTypeEndpoint), + }, + } +} + +func unsupportedAuthTypeError(serviceType ServiceType, authType internal.AuthType) error { + return fmt.Errorf("unsupported auth type, serviceType = %s, authType = %s", serviceType, authType) +} + +func mergeEnvWithDuplicationCheck(a []Env, b []Env) ([]Env, error) { + ab := append(a, b...) + var result []Env + seenName := make(map[string]Env) + for _, value := range ab { + if existingValue, exist := seenName[value.Name]; exist { + if value != existingValue { + return []Env{}, duplicatedEnvError(existingValue, value) + } + } else { + seenName[value.Name] = value + result = append(result, value) + } + } + return result, nil +} + +func duplicatedEnvError(existingValue Env, newValue Env) error { + return fmt.Errorf( + "duplicated environment variable. existingValue = %s, newValue = %s", + existingValue, newValue, + ) +} diff --git a/cli/azd/internal/scaffold/spec_service_binding_test.go b/cli/azd/internal/scaffold/spec_service_binding_test.go new file mode 100644 index 00000000000..fc47c352a1e --- /dev/null +++ b/cli/azd/internal/scaffold/spec_service_binding_test.go @@ -0,0 +1,168 @@ +package scaffold + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToServiceBindingEnvName(t *testing.T) { + tests := []struct { + name string + inputResourceType ServiceType + inputResourceInfoType ServiceBindingInfoType + want string + }{ + { + name: "postgres password", + inputResourceType: ServiceTypeDbPostgres, + inputResourceInfoType: ServiceBindingInfoTypePassword, + want: "$service.binding:db.postgres:password", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := ToServiceBindingEnvValue(tt.inputResourceType, tt.inputResourceInfoType) + assert.Equal(t, tt.want, actual) + }) + } +} + +func TestIsServiceBindingEnvName(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + { + name: "valid", + input: "$service.binding:db.postgres:password", + want: true, + }, + { + name: "invalid", + input: "$service.binding:db.postgres:", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isServiceBindingEnvValue(tt.input) + assert.Equal(t, tt.want, result) + }) + } +} + +func TestToServiceTypeAndServiceBindingInfoType(t *testing.T) { + tests := []struct { + name string + input string + wantResourceType ServiceType + wantResourceInfoType ServiceBindingInfoType + }{ + { + name: "invalid input", + input: "$service.binding:db.mysql::username", + wantResourceType: "", + wantResourceInfoType: "", + }, + { + name: "postgres password", + input: "$service.binding:db.postgres:password", + wantResourceType: ServiceTypeDbPostgres, + wantResourceInfoType: ServiceBindingInfoTypePassword, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resourceType, resourceInfoType := toServiceTypeAndServiceBindingInfoType(tt.input) + assert.Equal(t, tt.wantResourceType, resourceType) + assert.Equal(t, tt.wantResourceInfoType, resourceInfoType) + }) + } +} + +func TestMergeEnvWithDuplicationCheck(t *testing.T) { + var empty []Env + name1Value1 := []Env{ + { + Name: "name1", + Value: "value1", + }, + } + name1Value2 := []Env{ + { + Name: "name1", + Value: "value2", + }, + } + name2Value2 := []Env{ + { + Name: "name2", + Value: "value2", + }, + } + name1Value1Name2Value2 := []Env{ + { + Name: "name1", + Value: "value1", + }, + { + Name: "name2", + Value: "value2", + }, + } + + tests := []struct { + name string + a []Env + b []Env + wantEnv []Env + wantError error + }{ + { + name: "2 empty array", + a: empty, + b: empty, + wantEnv: empty, + wantError: nil, + }, + { + name: "one is empty, another is not", + a: empty, + b: name1Value1, + wantEnv: name1Value1, + wantError: nil, + }, + { + name: "no duplication", + a: name1Value1, + b: name2Value2, + wantEnv: name1Value1Name2Value2, + wantError: nil, + }, + { + name: "duplicated name but same value", + a: name1Value1, + b: name1Value1, + wantEnv: name1Value1, + wantError: nil, + }, + { + name: "duplicated name, different value", + a: name1Value1, + b: name1Value2, + wantEnv: []Env{}, + wantError: fmt.Errorf("duplicated environment variable. existingValue = %s, newValue = %s", + name1Value1[0], name1Value2[0]), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env, err := mergeEnvWithDuplicationCheck(tt.a, tt.b) + assert.Equal(t, tt.wantEnv, env) + assert.Equal(t, tt.wantError, err) + }) + } +} diff --git a/cli/azd/pkg/project/importer.go b/cli/azd/pkg/project/importer.go index 26fbde3a07e..3494d76b81c 100644 --- a/cli/azd/pkg/project/importer.go +++ b/cli/azd/pkg/project/importer.go @@ -167,7 +167,7 @@ func (im *ImportManager) ProjectInfrastructure(ctx context.Context, projectConfi composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) if composeEnabled && len(projectConfig.Resources) > 0 { - return tempInfra(ctx, projectConfig) + return tempInfra(ctx, projectConfig, im.dotNetImporter.console) } if !composeEnabled && len(projectConfig.Resources) > 0 { @@ -209,7 +209,7 @@ func (im *ImportManager) SynthAllInfrastructure(ctx context.Context, projectConf composeEnabled := im.dotNetImporter.alphaFeatureManager.IsEnabled(featureCompose) if composeEnabled && len(projectConfig.Resources) > 0 { - return infraFsForProject(ctx, projectConfig) + return infraFsForProject(ctx, projectConfig, im.dotNetImporter.console) } if !composeEnabled && len(projectConfig.Resources) > 0 { diff --git a/cli/azd/pkg/project/importer_test.go b/cli/azd/pkg/project/importer_test.go index 168e5c93261..9194d3e527f 100644 --- a/cli/azd/pkg/project/importer_test.go +++ b/cli/azd/pkg/project/importer_test.go @@ -392,6 +392,7 @@ resources: - api postgresdb: type: db.postgres + authType: password mongodb: type: db.mongo redis: @@ -405,11 +406,15 @@ func Test_ImportManager_ProjectInfrastructure_FromResources(t *testing.T) { im := &ImportManager{ dotNetImporter: &DotNetImporter{ alphaFeatureManager: alpha.NewFeaturesManagerWithConfig(config.NewEmptyConfig()), + console: mocks.NewMockContext(context.Background()).Console, }, } prjConfig := &ProjectConfig{} err := yaml.Unmarshal([]byte(prjWithResources), prjConfig) + for key, res := range prjConfig.Resources { + res.Name = key + } require.NoError(t, err) infra, err := im.ProjectInfrastructure(context.Background(), prjConfig) @@ -436,11 +441,15 @@ func TestImportManager_SynthAllInfrastructure_FromResources(t *testing.T) { im := &ImportManager{ dotNetImporter: &DotNetImporter{ alphaFeatureManager: alpha.NewFeaturesManagerWithConfig(config.NewEmptyConfig()), + console: mocks.NewMockContext(context.Background()).Console, }, } prjConfig := &ProjectConfig{} err := yaml.Unmarshal([]byte(prjWithResources), prjConfig) + for key, res := range prjConfig.Resources { + res.Name = key + } require.NoError(t, err) projectFs, err := im.SynthAllInfrastructure(context.Background(), prjConfig) diff --git a/cli/azd/pkg/project/resources.go b/cli/azd/pkg/project/resources.go index 9c1494ec15e..e1e8ec8846f 100644 --- a/cli/azd/pkg/project/resources.go +++ b/cli/azd/pkg/project/resources.go @@ -5,7 +5,7 @@ package project import ( "fmt" - + "github.com/azure/azure-dev/cli/azd/internal" "github.com/braydonk/yaml" ) @@ -89,6 +89,11 @@ func (r *ResourceConfig) MarshalYAML() (interface{}, error) { if err != nil { return nil, err } + case ResourceTypeDbPostgres: + err := marshalRawProps(raw.Props.(PostgresProps)) + if err != nil { + return nil, err + } } return raw, nil @@ -128,6 +133,12 @@ func (r *ResourceConfig) UnmarshalYAML(value *yaml.Node) error { return err } raw.Props = cap + case ResourceTypeDbPostgres: + pp := PostgresProps{} + if err := unmarshalProps(&pp); err != nil { + return err + } + raw.Props = pp } *r = ResourceConfig(raw) @@ -155,3 +166,8 @@ type AIModelPropsModel struct { Name string `yaml:"name,omitempty"` Version string `yaml:"version,omitempty"` } + +type PostgresProps struct { + DatabaseName string `yaml:"databaseName,omitempty"` + AuthType internal.AuthType `yaml:"authType,omitempty"` +} diff --git a/cli/azd/pkg/project/scaffold_gen.go b/cli/azd/pkg/project/scaffold_gen.go index 120f1c63211..84ee5b7e4fe 100644 --- a/cli/azd/pkg/project/scaffold_gen.go +++ b/cli/azd/pkg/project/scaffold_gen.go @@ -6,6 +6,7 @@ package project import ( "context" "fmt" + "github.com/azure/azure-dev/cli/azd/pkg/input" "io/fs" "os" "path/filepath" @@ -19,13 +20,13 @@ import ( ) // Generates the in-memory contents of an `infra` directory. -func infraFs(_ context.Context, prjConfig *ProjectConfig) (fs.FS, error) { +func infraFs(ctx context.Context, prjConfig *ProjectConfig, console input.Console) (fs.FS, error) { t, err := scaffold.Load() if err != nil { return nil, fmt.Errorf("loading scaffold templates: %w", err) } - infraSpec, err := infraSpec(prjConfig) + infraSpec, err := infraSpec(prjConfig, console, ctx) if err != nil { return nil, fmt.Errorf("generating infrastructure spec: %w", err) } @@ -41,13 +42,13 @@ func infraFs(_ context.Context, prjConfig *ProjectConfig) (fs.FS, error) { // Returns the infrastructure configuration that points to a temporary, generated `infra` directory on the filesystem. func tempInfra( ctx context.Context, - prjConfig *ProjectConfig) (*Infra, error) { + prjConfig *ProjectConfig, console input.Console) (*Infra, error) { tmpDir, err := os.MkdirTemp("", "azd-infra") if err != nil { return nil, fmt.Errorf("creating temporary directory: %w", err) } - files, err := infraFs(ctx, prjConfig) + files, err := infraFs(ctx, prjConfig, console) if err != nil { return nil, err } @@ -89,8 +90,8 @@ func tempInfra( // Generates the filesystem of all infrastructure files to be placed, rooted at the project directory. // The content only includes `./infra` currently. -func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, error) { - infraFS, err := infraFs(ctx, prjConfig) +func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig, console input.Console) (fs.FS, error) { + infraFS, err := infraFs(ctx, prjConfig, console) if err != nil { return nil, err } @@ -130,10 +131,8 @@ func infraFsForProject(ctx context.Context, prjConfig *ProjectConfig) (fs.FS, er return generatedFS, nil } -func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { +func infraSpec(projectConfig *ProjectConfig, console input.Console, ctx context.Context) (*scaffold.InfraSpec, error) { infraSpec := scaffold.InfraSpec{} - // backends -> frontends - backendMapping := map[string]string{} for _, res := range projectConfig.Resources { switch res.Type { @@ -147,23 +146,18 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { infraSpec.DbPostgres = &scaffold.DatabasePostgres{ DatabaseName: res.Name, DatabaseUser: "pgadmin", + AuthType: res.Props.(PostgresProps).AuthType, } case ResourceTypeHostContainerApp: svcSpec := scaffold.ServiceSpec{ Name: res.Name, Port: -1, } - err := mapContainerApp(res, &svcSpec, &infraSpec) if err != nil { return nil, err } - - err = mapHostUses(res, &svcSpec, backendMapping, projectConfig) - if err != nil { - return nil, err - } - + svcSpec.Envs = append(svcSpec.Envs, serviceConfigEnv(projectConfig.Services[res.Name])...) infraSpec.Services = append(infraSpec.Services, svcSpec) case ResourceTypeOpenAiModel: props := res.Props.(AIModelProps) @@ -185,18 +179,15 @@ func infraSpec(projectConfig *ProjectConfig) (*scaffold.InfraSpec, error) { } } - // create reverse frontends -> backends mapping - for i := range infraSpec.Services { - svc := &infraSpec.Services[i] - if front, ok := backendMapping[svc.Name]; ok { - if svc.Backend == nil { - svc.Backend = &scaffold.Backend{} - } - - svc.Backend.Frontends = append(svc.Backend.Frontends, scaffold.ServiceReference{Name: front}) - } + err := mapUses(&infraSpec, projectConfig) + if err != nil { + return nil, err } + err = printEnvListAboutUses(&infraSpec, projectConfig, console, ctx) + if err != nil { + return nil, err + } slices.SortFunc(infraSpec.Services, func(a, b scaffold.ServiceSpec) int { return strings.Compare(a.Name, b.Name) }) @@ -233,7 +224,10 @@ func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSp // Here, DB_HOST is not a secret, but DB_SECRET is. And yet, DB_HOST will be marked as a secret. // This is a limitation of the current implementation, but it's safer to mark both as secrets above. evaluatedValue := genBicepParamsFromEnvSubst(value, isSecret, infraSpec) - svcSpec.Env[envVar.Name] = evaluatedValue + err := scaffold.AddNewEnvironmentVariable(svcSpec, envVar.Name, evaluatedValue) + if err != nil { + return err + } } port := props.Port @@ -245,37 +239,97 @@ func mapContainerApp(res *ResourceConfig, svcSpec *scaffold.ServiceSpec, infraSp return nil } -func mapHostUses( - res *ResourceConfig, - svcSpec *scaffold.ServiceSpec, - backendMapping map[string]string, - prj *ProjectConfig) error { - for _, use := range res.Uses { - useRes, ok := prj.Resources[use] +func mapUses(infraSpec *scaffold.InfraSpec, projectConfig *ProjectConfig) error { + for i := range infraSpec.Services { + userSpec := &infraSpec.Services[i] + userResourceName := userSpec.Name + userResource, ok := projectConfig.Resources[userResourceName] if !ok { - return fmt.Errorf("resource %s uses %s, which does not exist", res.Name, use) + return fmt.Errorf("service (%s) exist, but there isn't a resource with that name", + userResourceName) } - - switch useRes.Type { - case ResourceTypeDbMongo: - svcSpec.DbCosmosMongo = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeDbPostgres: - svcSpec.DbPostgres = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeDbRedis: - svcSpec.DbRedis = &scaffold.DatabaseReference{DatabaseName: useRes.Name} - case ResourceTypeHostContainerApp: - if svcSpec.Frontend == nil { - svcSpec.Frontend = &scaffold.Frontend{} + for _, usedResourceName := range userResource.Uses { + usedResource, ok := projectConfig.Resources[usedResourceName] + if !ok { + return fmt.Errorf("in azure.yaml, (%s) uses (%s), but (%s) doesn't", + userResourceName, usedResourceName, usedResourceName) + } + var err error + switch usedResource.Type { + case ResourceTypeDbPostgres: + err = scaffold.BindToPostgres(userSpec, infraSpec.DbPostgres) + case ResourceTypeDbMongo: + err = scaffold.BindToMongoDb(userSpec, infraSpec.DbCosmosMongo) + case ResourceTypeDbRedis: + err = scaffold.BindToRedis(userSpec, infraSpec.DbRedis) + case ResourceTypeOpenAiModel: + err = scaffold.BindToAIModels(userSpec, usedResource.Name) + case ResourceTypeHostContainerApp: + usedSpec := getServiceSpecByName(infraSpec, usedResource.Name) + if usedSpec == nil { + return fmt.Errorf("'%s' uses '%s', but %s doesn't exist", userSpec.Name, usedResource.Name, + usedResource.Name) + } + scaffold.BindToContainerApp(userSpec, usedSpec) + default: + return fmt.Errorf("resource (%s) uses (%s), but the type of (%s) is (%s), which is unsupported", + userResource.Name, usedResource.Name, usedResource.Name, usedResource.Type) + } + if err != nil { + return err } - - svcSpec.Frontend.Backends = append(svcSpec.Frontend.Backends, - scaffold.ServiceReference{Name: use}) - backendMapping[use] = res.Name // record the backend -> frontend mapping - case ResourceTypeOpenAiModel: - svcSpec.AIModels = append(svcSpec.AIModels, scaffold.AIModelReference{Name: use}) } } + return nil +} +func printEnvListAboutUses(infraSpec *scaffold.InfraSpec, projectConfig *ProjectConfig, + console input.Console, ctx context.Context) error { + for i := range infraSpec.Services { + userSpec := &infraSpec.Services[i] + userResourceName := userSpec.Name + userResource, ok := projectConfig.Resources[userResourceName] + if !ok { + return fmt.Errorf("service (%s) exist, but there isn't a resource with that name", + userResourceName) + } + for _, usedResourceName := range userResource.Uses { + usedResource, ok := projectConfig.Resources[usedResourceName] + if !ok { + return fmt.Errorf("in azure.yaml, (%s) uses (%s), but (%s) doesn't", + userResourceName, usedResourceName, usedResourceName) + } + console.Message(ctx, fmt.Sprintf("\nInformation about environment variables:\n"+ + "In azure.yaml, '%s' uses '%s'. \n"+ + "The 'uses' relationship is implemented by environment variables. \n"+ + "Please make sure your application used the right environment variable. \n"+ + "Here is the list of environment variables: ", + userResourceName, usedResourceName)) + var variables []scaffold.Env + var err error + switch usedResource.Type { + case ResourceTypeDbPostgres: + variables, err = scaffold.GetServiceBindingEnvsForPostgres(*infraSpec.DbPostgres) + case ResourceTypeDbMongo: + variables = scaffold.GetServiceBindingEnvsForMongo() + case ResourceTypeDbRedis: + variables = scaffold.GetServiceBindingEnvsForRedis() + case ResourceTypeHostContainerApp: + printHintsAboutUseHostContainerApp(userResourceName, usedResourceName, console, ctx) + default: + return fmt.Errorf("resource (%s) uses (%s), but the type of (%s) is (%s), "+ + "which is doesn't add necessary environment variable", + userResource.Name, usedResource.Name, usedResource.Name, usedResource.Type) + } + if err != nil { + return err + } + for _, variable := range variables { + console.Message(ctx, fmt.Sprintf(" %s=xxx", variable.Name)) + } + console.Message(ctx, "\n") + } + } return nil } @@ -348,3 +402,37 @@ func genBicepParamsFromEnvSubst( return result } + +func getServiceSpecByName(infraSpec *scaffold.InfraSpec, name string) *scaffold.ServiceSpec { + for i := range infraSpec.Services { + if infraSpec.Services[i].Name == name { + return &infraSpec.Services[i] + } + } + return nil +} + +// todo: merge it into scaffold.BindToContainerApp +func printHintsAboutUseHostContainerApp(userResourceName string, usedResourceName string, + console input.Console, ctx context.Context) { + if console == nil { + return + } + console.Message(ctx, fmt.Sprintf("Environment variables in %s:", userResourceName)) + console.Message(ctx, fmt.Sprintf("%s_BASE_URL=xxx", strings.ToUpper(usedResourceName))) + console.Message(ctx, fmt.Sprintf("Environment variables in %s:", usedResourceName)) + console.Message(ctx, fmt.Sprintf("%s_BASE_URL=xxx", strings.ToUpper(userResourceName))) +} + +func serviceConfigEnv(svcConfig *ServiceConfig) []scaffold.Env { + var envs []scaffold.Env + if svcConfig != nil { + for key, val := range svcConfig.Env { + envs = append(envs, scaffold.Env{ + Name: key, + Value: val, + }) + } + } + return envs +} diff --git a/cli/azd/pkg/project/service_config.go b/cli/azd/pkg/project/service_config.go index aa3cf7bf640..f1cc057234e 100644 --- a/cli/azd/pkg/project/service_config.go +++ b/cli/azd/pkg/project/service_config.go @@ -45,6 +45,8 @@ type ServiceConfig struct { DotNetContainerApp *DotNetContainerAppOptions `yaml:"-,omitempty"` // Custom configuration for the service target Config map[string]any `yaml:"config,omitempty"` + // Environment variables for service + Env map[string]string `yaml:"env,omitempty"` // Computed lazily by useDotnetPublishForDockerBuild and cached. This is true when the project // is a dotnet project and there is not an explicit Dockerfile in the project directory. useDotNetPublishForDockerBuild *bool diff --git a/cli/azd/resources/scaffold/templates/resources.bicept b/cli/azd/resources/scaffold/templates/resources.bicept index 26180abdc28..4677cb45220 100644 --- a/cli/azd/resources/scaffold/templates/resources.bicept +++ b/cli/azd/resources/scaffold/templates/resources.bicept @@ -61,6 +61,15 @@ module containerAppsEnvironment 'br/public:avm/res/app/managed-environment:0.4.5 name: '${abbrs.appManagedEnvironments}${resourceToken}' location: location zoneRedundant: false + {{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity"))}} + roleAssignments: [ + { + principalId: connectionCreatorIdentity.outputs.principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + ] + {{- end}} } } {{- end}} @@ -101,8 +110,9 @@ module cosmos 'br/public:avm/res/document-db/database-account:0.8.1' = { {{- end}} {{- if .DbPostgres}} -var databaseName = '{{ .DbPostgres.DatabaseName }}' -var databaseUser = 'psqladmin' + +var postgreSqlDatabaseName = '{{ .DbPostgres.DatabaseName }}' +var postgreSqlDatabaseUser = '{{ .DbPostgres.DatabaseUser }}' module postgreServer 'br/public:avm/res/db-for-postgre-sql/flexible-server:0.1.4' = { name: 'postgreServer' params: { @@ -111,8 +121,8 @@ module postgreServer 'br/public:avm/res/db-for-postgre-sql/flexible-server:0.1.4 skuName: 'Standard_B1ms' tier: 'Burstable' // Non-required parameters - administratorLogin: databaseUser - administratorLoginPassword: databasePassword + administratorLogin: postgreSqlDatabaseUser + administratorLoginPassword: postgreSqlDatabasePassword geoRedundantBackup: 'Disabled' passwordAuth:'Enabled' firewallRules: [ @@ -124,13 +134,46 @@ module postgreServer 'br/public:avm/res/db-for-postgre-sql/flexible-server:0.1.4 ] databases: [ { - name: databaseName + name: postgreSqlDatabaseName } ] + } +} +{{- end}} + +{{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity"))}} + +module connectionCreatorIdentity 'br/public:avm/res/managed-identity/user-assigned-identity:0.2.1' = { + name: 'connectionCreatorIdentity' + params: { + name: '${abbrs.managedIdentityUserAssignedIdentities}cci-${resourceToken}' location: location } } {{- end}} +{{- range .Services}} +{{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity")) }} +var {{bicepName .Name}}PostgresConnectionName = 'connection_${uniqueString(subscription().id, resourceGroup().id, location, '{{bicepName .Name}}', 'Postgres')}' +module {{bicepName .Name}}CreateConnectionToPostgreSql 'br/public:avm/res/resources/deployment-script:0.4.0' = { + name: '{{bicepName .Name}}CreateConnectionToPostgreSql' + params: { + kind: 'AzureCLI' + name: '${abbrs.deploymentScript}{{bicepName .Name}}-connection-to-pg-${resourceToken}' + azCliVersion: '2.63.0' + location: location + {{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity")) }} + roleAssignments: [ + { + principalId: connectionCreatorIdentity.outputs.principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + ] + {{- end}} + } +} +{{- end}} +{{- end}} {{- if .AIModels}} var accountName = '${abbrs.cognitiveServicesAccounts}${resourceToken}' @@ -172,13 +215,22 @@ resource localUserOpenAIIdentity 'Microsoft.Authorization/roleAssignments@2022-0 } {{- end}} -{{- range .Services}} +{{- range $service := .Services}} module {{bicepName .Name}}Identity 'br/public:avm/res/managed-identity/user-assigned-identity:0.2.1' = { name: '{{bicepName .Name}}identity' params: { name: '${abbrs.managedIdentityUserAssignedIdentities}{{bicepName .Name}}-${resourceToken}' location: location + {{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity"))}} + roleAssignments: [ + { + principalId: connectionCreatorIdentity.outputs.principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + ] + {{- end}} } } @@ -234,41 +286,28 @@ module {{bicepName .Name}} 'br/public:avm/res/app/container-app:0.8.0' = { scaleMinReplicas: 1 scaleMaxReplicas: 10 secrets: { - secureList: union([ - {{- if .DbCosmosMongo}} - { - name: 'mongodb-url' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: cosmos.outputs.exportedSecrets['MONGODB-URL'].secretUri - } - {{- end}} - {{- if .DbPostgres}} - { - name: 'db-pass' - value: databasePassword - } - { - name: 'db-url' - value: 'postgresql://${databaseUser}:${databasePassword}@${postgreServer.outputs.fqdn}:5432/${databaseName}' - } - {{- end}} - {{- if .DbRedis}} - { - name: 'redis-pass' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: '${keyVault.outputs.uri}secrets/REDIS-PASSWORD' - } - { - name: 'redis-url' - identity:{{bicepName .Name}}Identity.outputs.resourceId - keyVaultUrl: '${keyVault.outputs.uri}secrets/REDIS-URL' - } - {{- end}} - ], - map({{bicepName .Name}}Secrets, secret => { - name: secret.secretRef - value: secret.value - })) + secureList: union([ + {{- range $env := .Envs}} + {{- if (eq (toBicepEnv $env).BicepEnvType "keyVaultSecret") }} + { + name: '{{ (toBicepEnv $env).SecretName }}' + identity:{{bicepName $service.Name}}Identity.outputs.resourceId + keyVaultUrl: {{ (toBicepEnv $env).SecretValue }} + } + {{- end}} + {{- if (eq (toBicepEnv $env).BicepEnvType "secret") }} + { + name: '{{ (toBicepEnv $env).SecretName }}' + value: {{ (toBicepEnv $env).SecretValue }} + } + {{- end}} + {{- end}} + ], + map({{bicepName .Name}}Secrets, secret => { + name: secret.secretRef + value: secret.value + }) + ) } containers: [ { @@ -293,32 +332,6 @@ module {{bicepName .Name}} 'br/public:avm/res/app/container-app:0.8.0' = { secretRef: 'mongodb-url' } {{- end}} - {{- if .DbPostgres}} - { - name: 'POSTGRES_HOST' - value: postgreServer.outputs.fqdn - } - { - name: 'POSTGRES_USERNAME' - value: databaseUser - } - { - name: 'POSTGRES_DATABASE' - value: databaseName - } - { - name: 'POSTGRES_PASSWORD' - secretRef: 'db-pass' - } - { - name: 'POSTGRES_URL' - secretRef: 'db-url' - } - { - name: 'POSTGRES_PORT' - value: '5432' - } - {{- end}} {{- if .DbRedis}} { name: 'REDIS_HOST' @@ -382,6 +395,15 @@ module {{bicepName .Name}} 'br/public:avm/res/app/container-app:0.8.0' = { environmentResourceId: containerAppsEnvironment.outputs.resourceId location: location tags: union(tags, { 'azd-service-name': '{{.Name}}' }) + {{- if (and .DbPostgres (eq .DbPostgres.AuthType "userAssignedManagedIdentity"))}} + roleAssignments: [ + { + principalId: connectionCreatorIdentity.outputs.principalId + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + } + ] + {{- end}} } } {{- end}} @@ -435,10 +457,10 @@ module keyVault 'br/public:avm/res/key-vault/vault:0.6.1' = { {{- end}} ] secrets: [ - {{- if .DbPostgres}} + {{- if (and .DbPostgres (eq .DbPostgres.AuthType "password")) }} { - name: 'db-pass' - value: databasePassword + name: 'postgresql-password' + value: postgreSqlDatabasePassword } {{- end}} ] diff --git a/cli/azd/test/functional/init_test.go b/cli/azd/test/functional/init_test.go index da748fa3e2b..3e4809947a5 100644 --- a/cli/azd/test/functional/init_test.go +++ b/cli/azd/test/functional/init_test.go @@ -203,6 +203,7 @@ func Test_CLI_Init_From_App_With_Infra(t *testing.T) { "Use code in the current directory\n"+ "Confirm and continue initializing my app\n"+ "appdb\n"+ + "User assigned managed identity\n"+ "TESTENV\n", "init", )