pangolin/.github/workflows/cicd.yml

name: CI/CD Pipeline

# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries.
# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events.

permissions:
  contents: read
  packages: write      # for GHCR push
  id-token: write      # for Cosign Keyless (OIDC) Signing

# Required secrets:
# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub
# - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing
# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing

on:
    push:
        tags:
            - "[0-9]+.[0-9]+.[0-9]+"
            - "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
    pre-run:
        runs-on: ubuntu-latest
        permissions: write-all
        steps:
            - name: Configure AWS credentials
              uses: aws-actions/configure-aws-credentials@v2
              with:
                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
                  role-duration-seconds: 3600
                  aws-region: ${{ secrets.AWS_REGION }}

            - name: Verify AWS identity
              run: aws sts get-caller-identity

            - name: Start EC2 instances
              run: |
                  aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
                  aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
                  echo "EC2 instances started"


    release-arm:
        name: Build and Release (ARM64)
        runs-on: [self-hosted, linux, arm64, us-east-1]
        needs: [pre-run]
        if: >-
            ${{
                needs.pre-run.result == 'success'
            }}
        # Job-level timeout to avoid runaway or stuck runs
        timeout-minutes: 120
        env:
          # Target images
          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

        steps:
            - name: Checkout code
              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

            - name: Monitor storage space
              run: |
                  THRESHOLD=75
                  USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
                  echo "Used space: $USED_SPACE%"
                  if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
                      echo "Used space is below the threshold of 75% free. Running Docker system prune."
                      echo y | docker system prune -a
                  else
                      echo "Storage space is above the threshold. No action needed."
                  fi

            - name: Log in to Docker Hub
              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
              with:
                  registry: docker.io
                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

            - name: Extract tag name
              id: get-tag
              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
              shell: bash

            - name: Update version in package.json
              run: |
                  TAG=${{ env.TAG }}
                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
                  cat server/lib/consts.ts
              shell: bash

            - name: Check if release candidate
              id: check-rc
              run: |
                  TAG=${{ env.TAG }}
                  if [[ "$TAG" == *"-rc."* ]]; then
                      echo "IS_RC=true" >> $GITHUB_ENV
                  else
                      echo "IS_RC=false" >> $GITHUB_ENV
                  fi
              shell: bash

            - name: Build and push Docker images (Docker Hub - ARM64)
              run: |
                  TAG=${{ env.TAG }}
                  if [ "$IS_RC" = "true" ]; then
                      make build-rc-arm tag=$TAG
                  else
                      make build-release-arm tag=$TAG
                  fi
                  echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
              shell: bash

    release-amd:
        name: Build and Release (AMD64)
        runs-on: [self-hosted, linux, x64, us-east-1]
        needs: [pre-run]
        if: >-
            ${{
                needs.pre-run.result == 'success'
            }}
        # Job-level timeout to avoid runaway or stuck runs
        timeout-minutes: 120
        env:
          # Target images
          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

        steps:
            - name: Checkout code
              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

            - name: Monitor storage space
              run: |
                  THRESHOLD=75
                  USED_SPACE=$(df / | grep / | awk '{ print $5 }' | sed 's/%//g')
                  echo "Used space: $USED_SPACE%"
                  if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
                      echo "Used space is below the threshold of 75% free. Running Docker system prune."
                      echo y | docker system prune -a
                  else
                      echo "Storage space is above the threshold. No action needed."
                  fi

            - name: Log in to Docker Hub
              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
              with:
                  registry: docker.io
                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

            - name: Extract tag name
              id: get-tag
              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
              shell: bash

            - name: Update version in package.json
              run: |
                  TAG=${{ env.TAG }}
                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
                  cat server/lib/consts.ts
              shell: bash

            - name: Check if release candidate
              id: check-rc
              run: |
                  TAG=${{ env.TAG }}
                  if [[ "$TAG" == *"-rc."* ]]; then
                      echo "IS_RC=true" >> $GITHUB_ENV
                  else
                      echo "IS_RC=false" >> $GITHUB_ENV
                  fi
              shell: bash

            - name: Build and push Docker images (Docker Hub - AMD64)
              run: |
                  TAG=${{ env.TAG }}
                  if [ "$IS_RC" = "true" ]; then
                      make build-rc-amd tag=$TAG
                  else
                      make build-release-amd tag=$TAG
                  fi
                  echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
              shell: bash

    create-manifest:
        name: Create Multi-Arch Manifests
        runs-on: [self-hosted, linux, x64, us-east-1]
        needs: [release-arm, release-amd]
        if: >-
            ${{
                needs.release-arm.result == 'success' &&
                needs.release-amd.result == 'success'
            }}
        timeout-minutes: 30
        steps:
            - name: Checkout code
              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

            - name: Log in to Docker Hub
              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
              with:
                  registry: docker.io
                  username: ${{ secrets.DOCKER_HUB_USERNAME }}
                  password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

            - name: Extract tag name
              id: get-tag
              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
              shell: bash

            - name: Check if release candidate
              id: check-rc
              run: |
                  TAG=${{ env.TAG }}
                  if [[ "$TAG" == *"-rc."* ]]; then
                      echo "IS_RC=true" >> $GITHUB_ENV
                  else
                      echo "IS_RC=false" >> $GITHUB_ENV
                  fi
              shell: bash

            - name: Create multi-arch manifests
              run: |
                  TAG=${{ env.TAG }}
                  if [ "$IS_RC" = "true" ]; then
                      make create-manifests-rc tag=$TAG
                  else
                      make create-manifests tag=$TAG
                  fi
                  echo "Created multi-arch manifests for tag: ${TAG}"
              shell: bash

    sign-and-package:
        name: Sign and Package
        runs-on: [self-hosted, linux, x64, us-east-1]
        needs: [release-arm, release-amd, create-manifest]
        if: >-
            ${{
                needs.release-arm.result == 'success' &&
                needs.release-amd.result == 'success' &&
                needs.create-manifest.result == 'success'
            }}
        # Job-level timeout to avoid runaway or stuck runs
        timeout-minutes: 120
        env:
          # Target images
          DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
          GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

        steps:
            - name: Checkout code
              uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

            - name: Extract tag name
              id: get-tag
              run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
              shell: bash

            - name: Install Go
              uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
              with:
                  go-version: 1.24

            - name: Update version in package.json
              run: |
                  TAG=${{ env.TAG }}
                  sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
                  cat server/lib/consts.ts
              shell: bash

            - name: Pull latest Gerbil version
              id: get-gerbil-tag
              run: |
                  LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags | jq -r '.[0].name')
                  echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV
              shell: bash

            - name: Pull latest Badger version
              id: get-badger-tag
              run: |
                  LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags | jq -r '.[0].name')
                  echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV
              shell: bash

            - name: Update install/main.go
              run: |
                  PANGOLIN_VERSION=${{ env.TAG }}
                  GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }}
                  BADGER_VERSION=${{ env.LATEST_BADGER_TAG }}
                  sed -i "s/config.PangolinVersion = \".*\"/config.PangolinVersion = \"$PANGOLIN_VERSION\"/" install/main.go
                  sed -i "s/config.GerbilVersion = \".*\"/config.GerbilVersion = \"$GERBIL_VERSION\"/" install/main.go
                  sed -i "s/config.BadgerVersion = \".*\"/config.BadgerVersion = \"$BADGER_VERSION\"/" install/main.go
                  echo "Updated install/main.go with Pangolin version $PANGOLIN_VERSION, Gerbil version $GERBIL_VERSION, and Badger version $BADGER_VERSION"
                  cat install/main.go
              shell: bash

            - name: Build installer
              working-directory: install
              run: |
                  make go-build-release

            - name: Upload artifacts from /install/bin
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              with:
                  name: install-bin
                  path: install/bin/

            - name: Install skopeo + jq
              # skopeo: copy/inspect images between registries
              # jq: JSON parsing tool used to extract digest values
              run: |
                  sudo apt-get update -y
                  sudo apt-get install -y skopeo jq
                  skopeo --version
              shell: bash

            - name: Login to GHCR
              env:
                  REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
              run: |
                  mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
                  skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}"
              shell: bash

            - name: Copy tag from Docker Hub to GHCR
              # Mirror the already-built image (all architectures) to GHCR so we can sign it
              # Wait a bit for both architectures to be available in Docker Hub manifest
              env:
                  REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
              run: |
                  set -euo pipefail
                  TAG=${{ env.TAG }}
                  echo "Waiting for multi-arch manifest to be ready..."
                  sleep 30
                  echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}"
                  skopeo copy --all --retry-times 3 \
                    docker://$DOCKERHUB_IMAGE:$TAG \
                    docker://$GHCR_IMAGE:$TAG
              shell: bash

            - name: Login to GitHub Container Registry (for cosign)
              uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
              with:
                registry: ghcr.io
                username: ${{ github.actor }}
                password: ${{ secrets.GITHUB_TOKEN }}

            - name: Install cosign
              # cosign is used to sign and verify container images (key and keyless)
              uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0

            - name: Dual-sign and verify (GHCR & Docker Hub)
              # Sign each image by digest using keyless (OIDC) and key-based signing,
              # then verify both the public key signature and the keyless OIDC signature.
              env:
                  TAG: ${{ env.TAG }}
                  COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
                  COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
                  COSIGN_PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
                  COSIGN_YES: "true"
              run: |
                  set -euo pipefail

                  issuer="https://token.actions.githubusercontent.com"
                  id_regex="^https://github.com/${{ github.repository }}/.+"  # accept this repo (all workflows/refs)

                  for IMAGE in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do
                    echo "Processing ${IMAGE}:${TAG}"

                    DIGEST="$(skopeo inspect --retry-times 3 docker://${IMAGE}:${TAG} | jq -r '.Digest')"
                    REF="${IMAGE}@${DIGEST}"
                    echo "Resolved digest: ${REF}"

                    echo "==> cosign sign (keyless) --recursive ${REF}"
                    cosign sign --recursive "${REF}"

                    echo "==> cosign sign (key) --recursive ${REF}"
                    cosign sign --key env://COSIGN_PRIVATE_KEY --recursive "${REF}"

                    echo "==> cosign verify (public key) ${REF}"
                    cosign verify --key env://COSIGN_PUBLIC_KEY "${REF}" -o text

                    echo "==> cosign verify (keyless policy) ${REF}"
                    cosign verify \
                      --certificate-oidc-issuer "${issuer}" \
                      --certificate-identity-regexp "${id_regex}" \
                      "${REF}" -o text
                  done
              shell: bash

    post-run:
        needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package]
        if: >-
            ${{
                always() &&
                needs.pre-run.result == 'success' &&
                (needs.release-arm.result == 'success' || needs.release-arm.result == 'skipped' || needs.release-arm.result == 'failure') &&
                (needs.release-amd.result == 'success' || needs.release-amd.result == 'skipped' || needs.release-amd.result == 'failure') &&
                (needs.create-manifest.result == 'success' || needs.create-manifest.result == 'skipped' || needs.create-manifest.result == 'failure') &&
                (needs.sign-and-package.result == 'success' || needs.sign-and-package.result == 'skipped' || needs.sign-and-package.result == 'failure')
            }}
        runs-on: ubuntu-latest
        permissions: write-all
        steps:
            - name: Configure AWS credentials
              uses: aws-actions/configure-aws-credentials@v2
              with:
                  role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
                  role-duration-seconds: 3600
                  aws-region: ${{ secrets.AWS_REGION }}

            - name: Verify AWS identity
              run: aws sts get-caller-identity

            - name: Stop EC2 instances
              run: |
                  aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
                  aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
                  echo "EC2 instances stopped"