From 57d3193b3d9d0f09d335053ca13d657db0d38a0a Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Sat, 2 Sep 2023 05:24:05 +0530 Subject: [PATCH] ci: upgrade uffizzi workflow to use clusters Signed-off-by: Vibhav Bobade --- .github/uffizzi/docker-compose.uffizzi.yml | 35 ---- .../uffizzi/k8s/manifests/backstage-crb.yaml | 12 ++ .../k8s/manifests/backstage-deploy.yaml | 37 ++++ .../k8s/manifests/backstage-ingress.yaml | 20 ++ .../uffizzi/k8s/manifests/backstage-sa.yaml | 5 + .../uffizzi/k8s/manifests/backstage-svc.yaml | 11 ++ .../uffizzi/k8s/manifests/kustomization.yaml | 12 ++ .github/uffizzi/k8s/manifests/pg-deploy.yaml | 31 +++ .github/uffizzi/k8s/manifests/pg-secret.yaml | 8 + .github/uffizzi/k8s/manifests/pg-svc.yaml | 9 + .github/uffizzi/k8s/manifests/pg-volume.yaml | 10 + .../uffizzi.production.app-config.yaml | 5 +- .github/workflows/uffizzi-build.yml | 26 +-- .github/workflows/uffizzi-preview.yaml | 182 +++++++++++++++--- 14 files changed, 331 insertions(+), 72 deletions(-) delete mode 100644 .github/uffizzi/docker-compose.uffizzi.yml create mode 100644 .github/uffizzi/k8s/manifests/backstage-crb.yaml create mode 100644 .github/uffizzi/k8s/manifests/backstage-deploy.yaml create mode 100644 .github/uffizzi/k8s/manifests/backstage-ingress.yaml create mode 100644 .github/uffizzi/k8s/manifests/backstage-sa.yaml create mode 100644 .github/uffizzi/k8s/manifests/backstage-svc.yaml create mode 100644 .github/uffizzi/k8s/manifests/kustomization.yaml create mode 100644 .github/uffizzi/k8s/manifests/pg-deploy.yaml create mode 100644 .github/uffizzi/k8s/manifests/pg-secret.yaml create mode 100644 .github/uffizzi/k8s/manifests/pg-svc.yaml create mode 100644 .github/uffizzi/k8s/manifests/pg-volume.yaml diff --git a/.github/uffizzi/docker-compose.uffizzi.yml b/.github/uffizzi/docker-compose.uffizzi.yml deleted file mode 100644 index f2e9bbeee6..0000000000 --- a/.github/uffizzi/docker-compose.uffizzi.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3' - -x-uffizzi: - ingress: - service: backstage - port: 7007 - -services: - backstage: - image: '${BACKSTAGE_IMAGE}' - - environment: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_USER: postgres - POSTGRES_PASSWORD: kiTMoTsiEuyQ43GrL4Hv - REF_NAME: ${GITHUB_SHA} - NODE_ENV: production - deploy: - resources: - limits: - memory: 500M - entrypoint: '/bin/sh' - command: - - '-c' - - "APP_CONFIG_app_baseUrl=$$UFFIZZI_URL APP_CONFIG_backend_baseUrl=$$UFFIZZI_URL APP_CONFIG_auth_environment='production' node packages/backend --config app-config.yaml" - - db: - image: postgres - environment: - POSTGRES_PASSWORD: kiTMoTsiEuyQ43GrL4Hv - deploy: - resources: - limits: - memory: 250M diff --git a/.github/uffizzi/k8s/manifests/backstage-crb.yaml b/.github/uffizzi/k8s/manifests/backstage-crb.yaml new file mode 100644 index 0000000000..a5dad3408a --- /dev/null +++ b/.github/uffizzi/k8s/manifests/backstage-crb.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: backstage-cluster-ro +subjects: + - namespace: backstage + kind: ServiceAccount + name: backstage-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:aggregate-to-view diff --git a/.github/uffizzi/k8s/manifests/backstage-deploy.yaml b/.github/uffizzi/k8s/manifests/backstage-deploy.yaml new file mode 100644 index 0000000000..9bf62e991a --- /dev/null +++ b/.github/uffizzi/k8s/manifests/backstage-deploy.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backstage +spec: + replicas: 1 + selector: + matchLabels: + app: backstage + template: + metadata: + labels: + app: backstage + spec: + serviceAccountName: backstage-service-account + containers: + - name: backstage + image: backstage + command: + - /bin/sh + args: + - '-c' + - "APP_CONFIG_app_baseUrl=$$UFFIZZI_URL APP_CONFIG_backend_baseUrl=$$UFFIZZI_URL APP_CONFIG_auth_environment='production' node packages/backend --config app-config.yaml" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 7007 + envFrom: + - secretRef: + name: postgres-secrets + env: + - name: POSTGRES_PORT + value: '5432' + - name: POSTGRES_HOST + value: 'postgres.default.svc.cluster.local' + - name: NODE_ENV + value: production diff --git a/.github/uffizzi/k8s/manifests/backstage-ingress.yaml b/.github/uffizzi/k8s/manifests/backstage-ingress.yaml new file mode 100644 index 0000000000..8a7f58c524 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/backstage-ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: backstage +spec: + ingressClassName: uffizzi + rules: + - host: backstage.example.com + http: + paths: + - backend: + service: + name: backstage + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - backstage.example.com diff --git a/.github/uffizzi/k8s/manifests/backstage-sa.yaml b/.github/uffizzi/k8s/manifests/backstage-sa.yaml new file mode 100644 index 0000000000..db0b5bde38 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/backstage-sa.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: backstage-service-account + namespace: default diff --git a/.github/uffizzi/k8s/manifests/backstage-svc.yaml b/.github/uffizzi/k8s/manifests/backstage-svc.yaml new file mode 100644 index 0000000000..1f4abc79b4 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/backstage-svc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: backstage +spec: + selector: + app: backstage + ports: + - name: http + port: 80 + targetPort: http diff --git a/.github/uffizzi/k8s/manifests/kustomization.yaml b/.github/uffizzi/k8s/manifests/kustomization.yaml new file mode 100644 index 0000000000..d4db0e97b2 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - backstage-deploy.yaml + - backstage-crb.yaml + - backstage-ingress.yaml + - backstage-sa.yaml + - backstage-svc.yaml + - pg-svc.yaml + - pg-deploy.yaml + - pg-secret.yaml + - pg-volume.yaml diff --git a/.github/uffizzi/k8s/manifests/pg-deploy.yaml b/.github/uffizzi/k8s/manifests/pg-deploy.yaml new file mode 100644 index 0000000000..354acd8e87 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/pg-deploy.yaml @@ -0,0 +1,31 @@ +# kubernetes/postgres.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13.2-alpine + imagePullPolicy: 'IfNotPresent' + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: postgres-secrets + volumeMounts: + - mountPath: /var/lib/postgresql + name: postgresdb + volumes: + - name: postgresdb + persistentVolumeClaim: + claimName: postgres-storage-claim diff --git a/.github/uffizzi/k8s/manifests/pg-secret.yaml b/.github/uffizzi/k8s/manifests/pg-secret.yaml new file mode 100644 index 0000000000..28a31f7c53 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/pg-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secrets +type: Opaque +data: + POSTGRES_USER: YmFja3N0YWdl + POSTGRES_PASSWORD: aHVudGVyMg== diff --git a/.github/uffizzi/k8s/manifests/pg-svc.yaml b/.github/uffizzi/k8s/manifests/pg-svc.yaml new file mode 100644 index 0000000000..ab7e192a65 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/pg-svc.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + selector: + app: postgres + ports: + - port: 5432 diff --git a/.github/uffizzi/k8s/manifests/pg-volume.yaml b/.github/uffizzi/k8s/manifests/pg-volume.yaml new file mode 100644 index 0000000000..0dcb317e50 --- /dev/null +++ b/.github/uffizzi/k8s/manifests/pg-volume.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-storage-claim +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2G diff --git a/.github/uffizzi/uffizzi.production.app-config.yaml b/.github/uffizzi/uffizzi.production.app-config.yaml index 34f4c90f0b..fb6638295e 100644 --- a/.github/uffizzi/uffizzi.production.app-config.yaml +++ b/.github/uffizzi/uffizzi.production.app-config.yaml @@ -160,7 +160,10 @@ kubernetes: type: 'multiTenant' clusterLocatorMethods: - type: 'config' - clusters: [] + clusters: + - url: ${UFFIZZI_CLUSTER_APISERVER} + name: uffizzi + authProvider: 'serviceAccount' kafka: clientId: backstage diff --git a/.github/workflows/uffizzi-build.yml b/.github/workflows/uffizzi-build.yml index 910cff0873..75eeb370ee 100644 --- a/.github/workflows/uffizzi-build.yml +++ b/.github/workflows/uffizzi-build.yml @@ -12,7 +12,7 @@ jobs: build-backstage: env: NODE_OPTIONS: --max-old-space-size=4096 - UFFIZZI_URL: https://uffizzi.com + UFFIZZI_URL: https://app.uffizzi.com name: Build PR image runs-on: ubuntu-latest if: ${{ github.event_name != 'pull_request' || github.event.action != 'closed' }} @@ -72,14 +72,14 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true + cache-from: type=gha + cache-to: type=gha,mode=max - render-compose-file: - name: Render Docker Compose File + render-kustomize: + name: Render Kustomize Manifests runs-on: ubuntu-latest needs: - build-backstage - outputs: - compose-file-cache-key: ${{ steps.hash.outputs.hash }} steps: - name: Harden Runner uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 @@ -90,16 +90,16 @@ jobs: uses: actions/checkout@v4.1.1 - name: Render Compose File run: | - BACKSTAGE_IMAGE=$(echo ${{ needs.build-backstage.outputs.tags }}) - export BACKSTAGE_IMAGE - # Render simple template from environment variables. - envsubst '$BACKSTAGE_IMAGE $GITHUB_SHA' < .github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml - cat docker-compose.rendered.yml - - name: Upload Rendered Compose File as Artifact - uses: actions/upload-artifact@v3.1.3 + # update image after the build above + cd ./.github/uffizzi/k8s/manifests + kustomize edit set image backstage=${{ needs.build-backstage.outputs.tags }} + kustomize build . > manifests.rendered.yml + cat manifests.rendered.yml + - name: Upload Rendered Manifests File as Artifact + uses: actions/upload-artifact@v3 with: name: preview-spec - path: docker-compose.rendered.yml + path: ./.github/uffizzi/k8s/manifests/manifests.rendered.yml retention-days: 2 - name: Upload PR Event as Artifact uses: actions/upload-artifact@v3.1.3 diff --git a/.github/workflows/uffizzi-preview.yaml b/.github/workflows/uffizzi-preview.yaml index deebf356bf..1d13680b82 100644 --- a/.github/workflows/uffizzi-preview.yaml +++ b/.github/workflows/uffizzi-preview.yaml @@ -8,14 +8,15 @@ on: - completed jobs: - cache-compose-file: - name: Cache Compose File + cache-manifests-file: + name: Cache Manifests File runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' }} outputs: - compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }} + manifests-cache-key: ${{ env.MANIFESTS_FILE_HASH }} git-ref: ${{ env.GIT_REF }} pr-number: ${{ env.PR_NUMBER }} + action: ${{ env.ACTION }} steps: - name: Harden Runner uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 @@ -56,23 +57,29 @@ jobs: cat event.json >> $GITHUB_ENV echo -e '\nEOF' >> $GITHUB_ENV - - name: Hash Rendered Compose File + - name: Hash Rendered Manifests File id: hash - # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact. + # If the previous workflow was triggered by a PR close event, we will not have a manifests file artifact. if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }} - run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV + run: | + ls + echo "MANIFESTS_FILE_HASH=$(md5sum manifests.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV - - name: Cache Rendered Compose File + - name: Cache Manifests File if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }} uses: actions/cache@v3.3.2 with: - path: docker-compose.rendered.yml - key: ${{ env.COMPOSE_FILE_HASH }} + path: manifests.rendered.yml + key: ${{ env.MANIFESTS_FILE_HASH }} - name: Read PR Number From Event Object id: pr run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV + - name: Read Event Type from Event Object + id: action + run: echo "ACTION=${{ fromJSON(env.EVENT_JSON).action }}" >> $GITHUB_ENV + - name: Read Git Ref From Event Object id: ref run: echo "GIT_REF=${{ fromJSON(env.EVENT_JSON).pull_request.head.sha }}" >> $GITHUB_ENV @@ -82,24 +89,153 @@ jobs: run: | echo "PR number: ${{ env.PR_NUMBER }}" echo "Git Ref: ${{ env.GIT_REF }}" - echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}" + echo "Manifests file hash: ${{ env.MANIFESTS_FILE_HASH }}" cat event.json deploy-uffizzi-preview: - name: Use Remote Workflow to Preview on Uffizzi - needs: - - cache-compose-file - if: ${{ github.event.workflow_run.conclusion == 'success' }} - uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2 - with: - # If this workflow was triggered by a PR close event, cache-key will be an empty string - # and this reusable workflow will delete the preview deployment. - compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }} - compose-file-cache-path: docker-compose.rendered.yml - git-ref: ${{ needs.cache-compose-file.outputs.git-ref }} - pr-number: ${{ needs.cache-compose-file.outputs.pr-number }} - server: https://app.uffizzi.com permissions: contents: read pull-requests: write id-token: write + name: Deploy to Uffizzi Virtual Cluster + needs: + - cache-manifests-file + if: ${{ github.event.workflow_run.conclusion == 'success' && needs.cache-manifests-file.outputs.action != 'closed' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Identify comment to be updated + - name: Find comment for Ephemeral Environment + uses: peter-evans/find-comment@v2 + id: find-comment + with: + issue-number: ${{ needs.cache-manifests-file.outputs.pr-number }} + comment-author: 'github-actions[bot]' + body-includes: pr-${{ needs.cache-manifests-file.outputs.pr-number }} + direction: last + + # Create/Update comment with action deployment status + - name: Create or Update Comment with Deployment Notification + id: notification + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ needs.cache-manifests-file.outputs.pr-number }} + body: | + ## Uffizzi Ephemeral Environment - Virtual Cluster + + :cloud: deploying ... + + :gear: Updating now by workflow run [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + + Download the Uffizzi CLI to interact with the upcoming virtual cluster + https://docs.uffizzi.com/install + edit-mode: replace + + - name: Connect to Virtual Cluster + uses: UffizziCloud/cluster-action@main + with: + cluster-name: pr-${{ needs.cache-manifests-file.outputs.pr-number }} + server: https://app.uffizzi.com + + - name: Fetch cached Manifests File + id: cache + # if: ${{ contains(fromJSON('["create", "update"]'), env.UFFIZZI_ACTION) }} + uses: actions/cache@v3 + with: + path: manifests.rendered.yml + key: ${{ needs.cache-manifests-file.outputs.manifests-cache-key }} + + - name: Kustomize and Apply Manifests + id: prev + run: | + # Apply kustomized manifests to virtual cluster. + export KUBECONFIG=`pwd`/kubeconfig + kubectl apply -f manifests.rendered.yml --kubeconfig ./kubeconfig + # Allow uffizzi to sync the resources + sleep 10 + # Get the hostnames assigned by uffizzi + export BACKSTAGE_HOST=$(kubectl get ingress backstage --kubeconfig kubeconfig -o json | jq '.spec.rules[0].host' | tr -d '"') + export UFFIZZI_CLUSTER_APISERVER=$(kubectl config view --minify | grep server | cut -f 2- -d ":" | tr -d " ") + # Patch backstage deployment to use UFFIZZI_URL + kubectl patch deployment backstage --kubeconfig kubeconfig -p '{"spec": {"template": {"spec": {"containers": [{"name": "backstage", "args":["-c", "APP_CONFIG_app_baseUrl='https://${BACKSTAGE_HOST}' APP_CONFIG_backend_baseUrl='https://${BACKSTAGE_HOST}' APP_CONFIG_auth_environment='production' node packages/backend --config app-config.yaml"], "env": [{"name": "UFFIZZI_URL", "value": "'https://${BACKSTAGE_HOST}'"}, {"name": "UFFIZZI_CLUSTER_APISERVER", "value": "'${UFFIZZI_CLUSTER_APISERVER}'"}, {"name": "GITHUB_SHA", "value": "'${GITHUB_SHA}'"}, {"name": "REF_NAME", "value": "'${{ needs.cache-manifests-file.outputs.git-ref }}'"}]}]}}}}' + if [[ ${RUNNER_DEBUG} == 1 ]]; then + kubectl get all --kubeconfig ./kubeconfig + fi + echo "backstage_url=${BACKSTAGE_HOST}" >> $GITHUB_OUTPUT + echo "Access the \`backstage\` endpoint at [\`${BACKSTAGE_HOST}\`](http://${BACKSTAGE_HOST})" >> $GITHUB_STEP_SUMMARY + + - name: Create or Update Comment with Deployment URL + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.notification.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ## Uffizzi Ephemeral Environment - Virtual Cluster + + Your cluster `pr-${{ needs.cache-manifests-file.outputs.pr-number }}` was successfully created. Learn more about [Uffizzi virtual clusters](https://docs.uffizzi.com/topics/virtual-clusters) + To connect to this cluster, follow these steps: + + 1. Download and install the Uffizzi CLI from https://docs.uffizzi.com/install + 2. Login to Uffizzi, then select the `backstage` account and project: + ``` + uffizzi login + ``` + + ``` + Select an account: + ‣ ${{ github.event.repository.name }} + jdoe + + Select a project or create a new project: + ‣ ${{ github.event.repository.name }}-6783521 + ``` + 3. Update your kubeconfig: `uffizzi cluster update-kubeconfig pr-${{ needs.cache-manifests-file.outputs.pr-number }} --kubeconfig=[PATH_TO_KUBECONFIG]` + After updating your kubeconfig, you can manage your cluster with `kubectl`, `kustomize`, `helm`, and other tools that use kubeconfig files: `kubectl get namespace --kubeconfig [PATH_TO_KUBECONFIG]` + + + Access the `backstage` endpoint at [`https://${{ steps.prev.outputs.backstage_url }}`](https://${{ steps.prev.outputs.backstage_url }}) + + edit-mode: replace + + delete-uffizzi-preview: + permissions: + contents: read + pull-requests: write + id-token: write + name: Delete the Uffizzi Virtual Cluster + needs: + - cache-manifests-file + if: ${{ needs.cache-manifests-file.outputs.action == 'closed' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Delete Virtual Cluster + uses: UffizziCloud/cluster-action@main + with: + cluster-name: pr-${{ needs.cache-manifests-file.outputs.pr-number }} + server: https://app.uffizzi.com + action: delete + + # Identify comment to be updated + - name: Find comment for Ephemeral Environment + uses: peter-evans/find-comment@v2 + id: find-comment + with: + issue-number: ${{ needs.cache-manifests-file.outputs.pr-number }} + comment-author: 'github-actions[bot]' + body-includes: pr-${{ needs.cache-manifests-file.outputs.pr-number }} + direction: last + + - name: Update Comment with Deletion + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ needs.cache-manifests-file.outputs.pr-number }} + body: | + Uffizzi Cluster `pr-${{ needs.cache-manifests-file.outputs.pr-number }}` was deleted. + edit-mode: replace