Deploy AKS LTS Prow #306
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy AKS LTS Prow | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| deployEnv: | |
| description: 'Environment to run against' | |
| type: environment | |
| required: true | |
| deploy_capz: | |
| description: 'Deploy CAPZ infrastructure (disabled — not testable right now)' | |
| type: boolean | |
| required: false | |
| default: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| Deploy_AKS_LTS_Prow: | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.deployEnv }} | |
| env: | |
| GITHUB_APP_ID: ${{ vars.APP_ID }} | |
| GITHUB_ORG: ${{ vars.ORG }} | |
| GITHUB_REPO: ${{ vars.REPO }} | |
| HMAC_TOKEN: ${{ secrets.HMAC_TOKEN }} | |
| MINIO_CONSOLE_PORT: 8003 | |
| K8S_PROW_IMAGE_TAG: v20260225-8287579aa | |
| steps: | |
| - name: Generate fake mount secret | |
| run: | | |
| FAKE_MOUNT_SECRET=$(echo '{"account":"fake","password":"fake"}' | base64) | |
| echo "::add-mask::$FAKE_MOUNT_SECRET" | |
| echo "FAKE_MOUNT_SECRET=$FAKE_MOUNT_SECRET" >> "$GITHUB_ENV" | |
| - name: Check out repo | |
| uses: actions/checkout@v6 | |
| - name: Azure login | |
| uses: azure/login@v3 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Create Prow resource group | |
| run: | | |
| if [ $(az group exists --name ${{ secrets.AZURE_RG }}) = false ]; then | |
| az group create --name ${{ secrets.AZURE_RG }} --location ${{ secrets.AZURE_LOCATION }} | |
| fi | |
| - name: Create CAPZ resource group | |
| if: inputs.deploy_capz | |
| run: | | |
| if [ $(az group exists --name ${{ secrets.CAPZ_RG }}) = false ]; then | |
| az group create --name ${{ secrets.CAPZ_RG }} --location ${{ secrets.AZURE_LOCATION }} | |
| fi | |
| - name: Generate FRONTDOOR_SECRET | |
| run: | | |
| FRONTDOOR_SECRET=$(openssl rand -base64 32) | |
| echo "::add-mask::$FRONTDOOR_SECRET" | |
| echo "FRONTDOOR_SECRET=$FRONTDOOR_SECRET" >> "$GITHUB_ENV" | |
| - name: Deploy Prow cluster Bicep | |
| id: bicep | |
| uses: azure/arm-deploy@v2 | |
| with: | |
| subscriptionId: ${{ vars.AZURE_SUBSCRIPTION_ID }} | |
| resourceGroupName: ${{ secrets.AZURE_RG }} | |
| template: ./config/prow/cluster/prow-cluster.bicep | |
| parameters: aks_cluster_region=${{ secrets.AZURE_LOCATION }} aks_cluster_admin_groups="${{ secrets.PROW_ADMIN_GROUPS }}" frontDoorSecret="${{ env.FRONTDOOR_SECRET }}" | |
| failOnStdErr: false | |
| - name: Deploy CAPZ Bicep | |
| if: inputs.deploy_capz | |
| id: capzbicep | |
| uses: azure/arm-deploy@v2 | |
| with: | |
| subscriptionId: ${{ vars.AZURE_SUBSCRIPTION_ID }} | |
| resourceGroupName: ${{ secrets.CAPZ_RG }} | |
| template: ./config/capz/capz.bicep | |
| parameters: location=${{ secrets.AZURE_LOCATION }} | |
| failOnStdErr: false | |
| - name: Fetch config | |
| run: | | |
| echo "PROW_HOST=${{ steps.bicep.outputs.prowHostName }}" >> "$GITHUB_ENV" | |
| echo "AZURE_STORAGE_ACCOUNT_USER=${{ steps.bicep.outputs.storageAccountName }}" >> "$GITHUB_ENV" | |
| echo "PUBLIC_IP_NAME=${{ steps.bicep.outputs.publicIpName }}" >> "$GITHUB_ENV" | |
| echo "CLUSTER_RG=${{ steps.bicep.outputs.resourceGroupName }}" >> "$GITHUB_ENV" | |
| - name: Fetch CAPZ config | |
| if: inputs.deploy_capz | |
| run: | | |
| echo "CAPZ_SA=${{ steps.capzbicep.outputs.capzsastorage_name }}" >> "$GITHUB_ENV" | |
| - name: Install Kubectl | |
| uses: azure/setup-kubectl@v5 | |
| # Use admin credentials temporarily to bootstrap RBAC for the GitHub OIDC identity | |
| - name: Set AKS cluster context (admin bootstrap) | |
| uses: azure/aks-set-context@v5 | |
| with: | |
| resource-group: ${{ secrets.AZURE_RG }} | |
| cluster-name: ${{ steps.bicep.outputs.aksClusterName }} | |
| admin: 'true' | |
| - name: Grant cluster-admin to GitHub OIDC identity | |
| run: | | |
| set -e | |
| OBJ_ID=$(az ad sp show --id ${{ secrets.AZURE_CLIENT_ID }} --query id -o tsv 2>/dev/null || echo ${{ secrets.AZURE_CLIENT_ID }}) | |
| echo "Using object id: $OBJ_ID" | |
| if ! kubectl get clusterrolebinding prow-github-oidc-admin >/dev/null 2>&1; then | |
| kubectl create clusterrolebinding prow-github-oidc-admin --clusterrole=cluster-admin --user=$OBJ_ID | |
| else | |
| echo "ClusterRoleBinding prow-github-oidc-admin already exists" | |
| fi | |
| - name: Set up kubelogin for non-interactive login | |
| uses: azure/use-kubelogin@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set AKS cluster context (oidc) | |
| uses: azure/aks-set-context@v5 | |
| with: | |
| resource-group: ${{ secrets.AZURE_RG }} | |
| cluster-name: ${{ steps.bicep.outputs.aksClusterName }} | |
| admin: 'false' | |
| use-kubelogin: 'true' | |
| - name: 'Apply Prow base manifests' | |
| run: | | |
| kubectl apply -f config/prow/k8s/base/ns.yaml | |
| envsubst < config/prow/k8s/base/contour.yaml > contour.yaml | |
| kubectl apply -f contour.yaml | |
| envsubst < config/prow/k8s/base/prowdata.storageclass.yaml > prowdata.storageclass.yaml | |
| kubectl apply -f prowdata.storageclass.yaml | |
| env: | |
| AZURE_RG: ${{ secrets.AZURE_RG }} | |
| - name: 'Create GitHub Token secrets' | |
| run: | | |
| echo "${{ secrets.APP_PRIVATE_KEY }}" > cert.pem | |
| kubectl create secret generic github-token -n prow --from-file=cert=cert.pem --from-literal=appid=$GITHUB_APP_ID -o yaml --dry-run=client | kubectl apply -f - | |
| kubectl create secret generic github-token -n test-pods --from-file=cert=cert.pem --from-literal=appid=$GITHUB_APP_ID -o yaml --dry-run=client | kubectl apply -f - | |
| rm cert.pem | |
| - name: 'Setup GitHub Oauth Secrets' | |
| run: | | |
| echo "client_id: ${{ vars.CLIENT_ID }}" > github-oauth-config-file | |
| echo "client_secret: ${{ secrets.APP_CLIENT_SECRET }}" >> github-oauth-config-file | |
| echo "redirect_url: https://$PROW_HOST/github-login/redirect" >> github-oauth-config-file | |
| echo "final_redirect_url: https://$PROW_HOST/pr" >> github-oauth-config-file | |
| echo "redirect_url: https://$PROW_HOST/github-login/redirect" | |
| echo "final_redirect_url: https://$PROW_HOST/pr" | |
| kubectl create secret generic github-oauth-config -n prow --from-file=secret=github-oauth-config-file -o yaml --dry-run=client | kubectl apply -f - | |
| kubectl create secret generic github-oauth-config -n test-pods --from-file=secret=github-oauth-config-file -o yaml --dry-run=client | kubectl apply -f - | |
| rm github-oauth-config-file | |
| openssl rand -out cookie.txt -base64 32 | |
| kubectl create secret generic cookie -n prow --from-file=secret=cookie.txt -o yaml --dry-run=client | kubectl apply -f - | |
| kubectl create secret generic cookie -n test-pods --from-file=secret=cookie.txt -o yaml --dry-run=client | kubectl apply -f - | |
| rm cookie.txt | |
| - name: Fetch storage key | |
| id: fetch-storage-key | |
| run: | | |
| AZURE_STORAGE_ACCOUNT_PASSWORD=$(az storage account keys list -g ${{ secrets.AZURE_RG }} -n ${{ steps.bicep.outputs.storageAccountName }} | jq -r '.[0].value') | |
| echo "::add-mask::$AZURE_STORAGE_ACCOUNT_PASSWORD" | |
| echo "AZURE_STORAGE_ACCOUNT_PASSWORD=$AZURE_STORAGE_ACCOUNT_PASSWORD" >> "$GITHUB_ENV" | |
| PUBLIC_IP_ADDRESS=$(az network public-ip show -g ${{ secrets.AZURE_RG }} -n ${{ steps.bicep.outputs.publicIpName }} | jq -r '.ipAddress') | |
| echo "::add-mask::$PUBLIC_IP_ADDRESS" | |
| echo "PUBLIC_IP_ADDRESS=$PUBLIC_IP_ADDRESS" >> "$GITHUB_ENV" | |
| echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> "$GITHUB_ENV" | |
| - name: Fetch CAPZ registry and key vault config | |
| if: inputs.deploy_capz | |
| run: | | |
| CAPZ_CI_REGISTRY=$(az acr show -g ${{ secrets.CAPZ_RG }} -n ${{ steps.capzbicep.outputs.capzci_registry_name }} | jq .loginServer) | |
| echo "::add-mask::$CAPZ_CI_REGISTRY" | |
| echo "CAPZ_CI_REGISTRY=$CAPZ_CI_REGISTRY" >> "$GITHUB_ENV" | |
| echo "::add-mask::${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" | |
| echo "CAPZ_GMSA_KV=${{ steps.capzbicep.outputs.capz_gmsa_kv_name }}" >> "$GITHUB_ENV" | |
| - name: 'Create job configs' | |
| run: | | |
| envsubst < config/prow/release-branch-jobs/base.yaml > cm.yaml | |
| for f in $(ls config/prow/release-branch-jobs/*.yaml | grep -v base.yaml | sort -V); do | |
| echo " Adding job config: $f" | |
| envsubst < "$f" >> cm.yaml | |
| done | |
| kubectl create configmap config -n prow --from-file=config.yaml=cm.yaml -o yaml --dry-run=client | kubectl apply -f - | |
| rm cm.yaml | |
| env: | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: 'Apply Prowjob CRD' | |
| run: for f in config/prow/k8s/prowjob/*.yaml; do kubectl apply --server-side=true -f $f; done | |
| - name: 'Apply Prow app manifests' | |
| run: for f in config/prow/k8s/app/*.yaml; do envsubst < $f | kubectl apply -f -; done | |
| env: | |
| AZURE_RG: ${{ secrets.AZURE_RG }} | |
| - name: 'Apply test pod manifests' | |
| run: for f in config/prow/k8s/test-pods/*.yaml; do envsubst < $f | kubectl apply -f -; done | |
| env: | |
| AZURE_RG: ${{ secrets.AZURE_RG }} | |
| - name: 'Restart Prow components if secrets are out of sync' | |
| run: | | |
| # Compute current secret hash | |
| SECRET_HASH=$(kubectl get secret s3-credentials -n prow -o jsonpath='{.data}' | md5sum | cut -d' ' -f1) | |
| echo "Current s3-credentials hash: $SECRET_HASH" | |
| # Check if Deck pods have the matching hash annotation | |
| DECK_HASH=$(kubectl get deployment deck -n prow -o jsonpath='{.spec.template.metadata.annotations.s3-credentials-hash}' 2>/dev/null || echo "") | |
| echo "Deck annotation hash: ${DECK_HASH:-<none>}" | |
| if [ "$SECRET_HASH" != "$DECK_HASH" ]; then | |
| echo "Secret hash mismatch — restarting Prow components..." | |
| # Annotate deployments with the current hash (triggers rollout) | |
| for deploy in deck crier; do | |
| kubectl patch deployment "$deploy" -n prow -p \ | |
| "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"s3-credentials-hash\":\"$SECRET_HASH\"}}}}}" | |
| done | |
| kubectl rollout status deployment -n prow deck crier --timeout=300s | |
| else | |
| echo "Secret hash matches — no restart needed" | |
| fi |