Skip to content

Commit

Permalink
CS pipeline.yaml
Browse files Browse the repository at this point in the history
Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
geoberle committed Dec 20, 2024
1 parent 0bd4db8 commit 896393f
Show file tree
Hide file tree
Showing 15 changed files with 380 additions and 141 deletions.
42 changes: 28 additions & 14 deletions cluster-service/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
SHELL = /bin/bash
DEPLOY_ENV ?= personal-dev
$(shell ../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk)
include config.mk
-include ../setup-env.mk

ZONE_NAME ?= "${REGIONAL_DNS_SUBDOMAIN}.${BASE_DNS_ZONE_NAME}"


deploy: provision-shard
@ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${RESOURCEGROUP} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \
Expand All @@ -14,8 +14,17 @@ deploy: provision-shard
OIDC_ISSUER_BASE_ENDPOINT=$(shell az storage account show -n ${OIDC_STORAGE_ACCOUNT} -g ${RESOURCEGROUP} --query primaryEndpoints.web -o tsv) && \
OCP_ACR_URL=$(shell az acr show -n ${OCP_ACR_NAME} --query loginServer -o tsv) && \
OCP_ACR_RESOURCE_ID=$(shell az acr show -n ${OCP_ACR_NAME} --query id -o tsv) && \
helm upgrade --install cluster-service --namespace cluster-service \
deploy/helm/ \
DB_HOST=$$(if [ "${USE_AZURE_DB}" = "true" ]; then az postgres flexible-server show -g ${RESOURCEGROUP} -n ${DATABASE_SERVER_NAME} --query fullyQualifiedDomainName -o tsv; else echo "ocm-cs-db"; fi) && \
OVERRIDES=$$(if [ "${USE_AZURE_DB}" = "true" ]; then echo "azuredb.values.yaml"; else echo "containerdb.values.yaml"; fi) && \
OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_INGRESS_ROLE_ID=$(shell az role definition list --name "${OP_INGRESS_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_DISK_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_DISK_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_FILE_CSI_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_FILE_CSI_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_IMAGE_REGISTRY_DRIVER_ROLE_ID=$(shell az role definition list --name "${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" --query "[].name" -o tsv) && \
OP_CLOUD_NETWORK_CONFIG_ROLE_ID=$(shell az role definition list --name "${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" --query "[].name" -o tsv) && \
helm upgrade --install --wait ${HELM_DRY_RUN} cluster-service deploy/helm \
--namespace cluster-service \
-f deploy/helm/$${OVERRIDES} \
--set azureCsMiClientId=$${AZURE_CS_MI_CLIENT_ID} \
--set oidcIssuerBlobServiceUrl=$${OIDC_BLOB_SERVICE_ENDPOINT} \
--set oidcIssuerBaseUrl=$${OIDC_ISSUER_BASE_ENDPOINT} \
Expand All @@ -30,21 +39,26 @@ deploy: provision-shard
--set fpaCertName=${FPA_CERT_NAME} \
--set ocpAcrResourceId=$${OCP_ACR_RESOURCE_ID} \
--set ocpAcrUrl=$${OCP_ACR_URL} \
--set databaseDisableTls=${DATABASE_DISABLE_TLS} \
--set databaseAuthMethod=${DATABASE_AUTH_METHOD} \
--set provisionShardsConfig="$(shell base64 -i deploy/provisioning-shards.yml | tr -d '\n')" \
--set deployLocalDatabase=${DEPLOY_LOCAL_DB} \
--set databaseHost=${DB_HOST} \
--set databaseName=${DB_NAME} \
--set databaseUser=${DB_USERNAME} \
--set databasePassword=${DB_PASSWORD} \
--set databaseHost=$${DB_HOST} \
--set azureMiMockServicePrincipalPrincipalId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID} \
--set azureMiMockServicePrincipalClientId=${AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID} \
--set azureMiMockServicePrincipalCertName=${MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME} \
--set azureArmHelperIdentityCertName=${ARM_HELPER_CERT_NAME} \
--set azureArmHelperIdentityClientId=${AZURE_ARM_HELPER_IDENTITY_CLIENT_ID} \
--set azureArmHelperMockFpaPrincipalId=${AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID} \
--set azureOperatorsManagedIdentitiesConfig=${AZURE_OPERATORS_MANAGED_IDENTITIES_CONFIG}
--set azureOperatorsMI.cloudControllerManager.roleName="${OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME}" \
--set azureOperatorsMI.cloudControllerManager.roleId="$${OP_CLOUD_CONTROLLER_MANAGER_ROLE_ID}" \
--set azureOperatorsMI.ingress.roleName="${OP_INGRESS_ROLE_NAME}" \
--set azureOperatorsMI.ingress.roleId="$${OP_INGRESS_ROLE_ID}" \
--set azureOperatorsMI.diskCsiDriver.roleName="${OP_DISK_CSI_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.diskCsiDriver.roleId="$${OP_DISK_CSI_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.fileCsiDriver.roleName="${OP_FILE_CSI_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.fileCsiDriver.roleId="$${OP_FILE_CSI_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.imageRegistry.roleName="${OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME}" \
--set azureOperatorsMI.imageRegistry.roleId="$${OP_IMAGE_REGISTRY_DRIVER_ROLE_ID}" \
--set azureOperatorsMI.cloudNetworkConfig.roleName="${OP_CLOUD_NETWORK_CONFIG_ROLE_NAME}" \
--set azureOperatorsMI.cloudNetworkConfig.roleId="$${OP_CLOUD_NETWORK_CONFIG_ROLE_ID}"

deploy-pr-env-deps:
AZURE_CS_MI_CLIENT_ID=$(shell az identity show -g ${RESOURCEGROUP} -n clusters-service --query clientId -o tsv) && \
Expand Down
39 changes: 0 additions & 39 deletions cluster-service/config.tmpl.mk

This file was deleted.

6 changes: 6 additions & 0 deletions cluster-service/deploy/helm/azuredb.values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
databaseDisableTls: false
databaseAuthMethod: az-entra
deployLocalDatabase: false
databaseName: cluseters-service
databaseUser: clusters-service
databasePassword: ''
6 changes: 6 additions & 0 deletions cluster-service/deploy/helm/containerdb.values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
databaseDisableTls: true
databaseAuthMethod: postgres
deployLocalDatabase: true
databaseName: ocm-cs-db
databaseUser: ocm
databasePassword: TheBlurstOfTimes
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,83 @@ metadata:
namespace: {{ .Release.Namespace }}
data:
azure-operators-managed-identities-config.yaml: |
{{ .Values.azureOperatorsManagedIdentitiesConfig | b64dec | indent 4 }}
controlPlaneOperatorsIdentities:
cloud-controller-manager:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudControllerManager.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudControllerManager.roleName }}'
optional: false
ingress:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
optional: false
disk-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
optional: false
file-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
optional: false
image-registry:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
optional: false
cloud-network-config:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
optional: false
dataPlaneOperatorsIdentities:
disk-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.diskCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.diskCsiDriver.roleName }}'
k8sServiceAccounts:
- name: 'azure-disk-csi-driver-operator'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-disk-csi-driver-controller-sa'
namespace: 'openshift-cluster-csi-drivers'
optional: false
image-registry:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.imageRegistry.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.imageRegistry.roleName }}'
k8sServiceAccounts:
- name: 'cluster-image-registry-operator'
namespace: 'openshift-image-registry'
- name: 'registry'
namespace: 'openshift-image-registry'
optional: false
file-csi-driver:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.fileCsiDriver.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.fileCsiDriver.roleName }}'
k8sServiceAccounts:
- name: 'azure-file-csi-driver-operator'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-file-csi-driver-controller-sa'
namespace: 'openshift-cluster-csi-drivers'
- name: 'azure-file-csi-driver-node-sa'
namespace: 'openshift-cluster-csi-drivers'
optional: false
ingress:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.ingress.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.ingress.roleName }}'
k8sServiceAccounts:
- name: 'ingress-operator'
namespace: 'openshift-ingress-operator'
optional: false
cloud-network-config:
minOpenShiftVersion: 4.17
azureRoleDefinitionResourceId: '/providers/Microsoft.Authorization/roleDefinitions/{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleId }}'
azureRoleDefinitionName: '{{ .Values.azureOperatorsMI.cloudNetworkConfig.roleName }}'
k8sServiceAccounts:
- name: 'cloud-network-config-controller'
namespace: 'openshift-cloud-network-config-controller'
optional: false
20 changes: 19 additions & 1 deletion cluster-service/deploy/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -262,4 +262,22 @@ databasePort: "5432"
managedIdentitiesDataPlaneAudienceResource: "https://dummy.org"

# The Azure Operator Managed Identities.
azureOperatorsManagedIdentitiesConfig: ""
azureOperatorsMI:
cloudControllerManager:
roleName: ''
roleId: ''
ingress:
roleName: ''
roleId: ''
diskCsiDriver:
roleName: ''
roleId: ''
fileCsiDriver:
roleName: ''
roleId: ''
imageRegistry:
roleName: ''
roleId: ''
cloudNetworkConfig:
roleName: ''
roleId: ''
87 changes: 87 additions & 0 deletions cluster-service/pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.ClusterService
rolloutName: Cluster Service Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
aksCluster: {{ .aksName }}
steps:
- name: deploy
action: Shell
command: make deploy
dryRun:
variables:
- name: HELM_DRY_RUN
value: "--dry-run=server --debug"
- name: KUBECTL_DRY_RUN
value: "--dry-run=server"
variables:
- name: REGION
configRef: region
- name: RESOURCEGROUP
configRef: svc.rg
- name: AKS_NAME
configRef: aksName
- name: SERVICE_KV
configRef: serviceKeyVault.name
- name: OIDC_STORAGE_ACCOUNT
configRef: oidcStorageAccountName
- name: IMAGE_REPO
configRef: clusterService.imageRepo
- name: IMAGE_TAG
configRef: clusterService.imageTag
- name: ACR_NAME
configRef: svcAcrName
- name: OCP_ACR_NAME
configRef: ocpAcrName
- name: AZURE_FIRST_PARTY_APPLICATION_CLIENT_ID
configRef: firstPartyAppClientId
- name: FPA_CERT_NAME
value: firstPartyCert
- name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_PRINCIPAL_ID
configRef: miMockPrincipalId
- name: AZURE_MI_MOCK_SERVICE_PRINCIPAL_CLIENT_ID
configRef: miMockClientId
- name: AZURE_ARM_HELPER_IDENTITY_CLIENT_ID
configRef: armHelperClientId
- name: AZURE_ARM_HELPER_MOCK_FPA_PRINCIPAL_ID
configRef: armHelperFPAPrincipalId
- name: MI_MOCK_SERVICE_PRINCIPAL_CERT_NAME
value: msiMockCert
- name: ARM_HELPER_CERT_NAME
value: armHelperCert
- name: BASE_DNS_ZONE_NAME
configRef: baseDnsZoneName
- name: REGIONAL_DNS_SUBDOMAIN
configRef: regionalDNSSubdomain
- name: USE_AZURE_DB
configRef: clusterService.postgres.deploy
- name: DATABASE_SERVER_NAME
configRef: clusterService.postgres.name
- name: DEVOPS_MSI_ID
configRef: aroDevopsMsiId
- name: OP_CLOUD_CONTROLLER_MANAGER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.cloudControllerManager.roleName
- name: OP_INGRESS_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.ingress.roleName
- name: OP_DISK_CSI_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.diskCsiDriver.roleName
- name: OP_FILE_CSI_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.fileCsiDriver.roleName
- name: OP_IMAGE_REGISTRY_DRIVER_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.imageRegistry.roleName
- name: OP_CLOUD_NETWORK_CONFIG_ROLE_NAME
configRef: clusterService.azureOperatorsManagedIdentities.cloudNetworkConfig.roleName

# this is maestro consumer registration stuff
# this goes away when we have a real registration process
- name: CONSUMER_NAME
configRef: maestro.consumerName
- name: REGIONAL_RESOURCEGROUP
configRef: regionRG
- name: MGMT_RESOURCEGROUP
configRef: mgmt.rg
- name: CX_SECRETS_KV_NAME
configRef: cxKeyVault.name
- name: CX_MI_KV_NAME
configRef: msiKeyVault.name
13 changes: 13 additions & 0 deletions config/config.msft.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,19 @@ clouds:
clusterService:
imageTag: ecd15ad
imageRepo: app-sre/uhc-clusters-service
azureOperatorsManagedIdentities:
cloudControllerManager:
roleName: Azure Red Hat OpenShift Cloud Controller Manager Role
ingress:
roleName: Azure Red Hat OpenShift Cluster Ingress Operator Role
diskCsiDriver:
roleName: Azure Red Hat OpenShift Disk Storage Operator Role
fileCsiDriver:
roleName: Azure Red Hat OpenShift File Storage Operator Role
imageRegistry:
roleName: Azure Red Hat OpenShift Image Registry Operator Role
cloudNetworkConfig:
roleName: Azure Red Hat OpenShift Network Operator Role
hypershiftOperator:
imageTag: 9aca808
imageSync:
Expand Down
47 changes: 45 additions & 2 deletions config/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,20 @@
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Generated schema for Root",
"type": "object",
"definitions": {
"operatorConfig": {
"type": "object",
"properties": {
"roleName": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"roleName"
]
}
},
"properties": {
"aksName": {
"type": "string"
Expand Down Expand Up @@ -55,8 +69,37 @@
"minTLSVersion"
]
},
"azureOperatorsManagedIdentitiesConfig":{
"type": "string"
"azureOperatorsManagedIdentities": {
"type": "object",
"properties": {
"cloudControllerManager": {
"$ref": "#/definitions/operatorConfig"
},
"ingress": {
"$ref": "#/definitions/operatorConfig"
},
"diskCsiDriver": {
"$ref": "#/definitions/operatorConfig"
},
"fileCsiDriver": {
"$ref": "#/definitions/operatorConfig"
},
"imageRegistry": {
"$ref": "#/definitions/operatorConfig"
},
"cloudNetworkConfig": {
"$ref": "#/definitions/operatorConfig"
}
},
"additionalProperties": false,
"required": [
"cloudControllerManager",
"ingress",
"diskCsiDriver",
"fileCsiDriver",
"imageRegistry",
"cloudNetworkConfig"
]
}
},
"additionalProperties": false,
Expand Down
Loading

0 comments on commit 896393f

Please sign in to comment.