Merge pull request #69 from pi-hole/ftl-build/buildx

Use docker buildx for ftl-build containers

.github/actions/login-repo/action.yml

@@ -0,0 +1,35 @@
+name: Login to container registries
+description: Login to container registries Docker Hub and GitHub Container Registry
+
+inputs:
+  # Actions cannot access secrets so pass them in as inputs
+  docker_username:
+    required: true
+    description: The username to use to login to Docker Hub
+  docker_password:
+    required: true
+    description: The password to use to login to Docker Hub
+  ghcr_username:
+    required: true
+    description: The username to use to login to GitHub Container Registry
+  ghcr_password:
+    required: true
+    description: The password to use to login to GitHub Container Registry
+
+runs:
+  using: "composite"
+  steps:
+    -
+      name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        registry: docker.io
+        username: ${{ inputs.docker_username }}
+        password: ${{ inputs.docker_password }}
+    -
+      name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.ghcr_username }}
+        password: ${{ inputs.ghcr_password }}

.github/actions/merge-and-push/action.yml

@@ -0,0 +1,50 @@
+name: Merge and push
+description: Apply meta, create manifest, and push to container registry
+
+inputs:
+  imagename:
+    required: true
+    description: The name of the image to push
+  platform:
+    required: true
+    description: The platform to push the image for
+  registry:
+    required: true
+    description: The Registry to push the image to
+
+runs:
+  using: "composite"
+  steps:
+    -
+      name: Docker meta
+      id: meta_docker
+      uses: docker/metadata-action@v4
+      with:
+        images: |
+          ${{ inputs.imagename }},enable=${{ github.event_name != 'workflow_dispatch' }}
+        # We want to tag the image with the latest tag if the workflow was triggered by a tag
+        flavor: |
+          latest=${{ startsWith(github.ref, 'refs/tags/') }}
+        # tags:
+          # type=schedule means that a tag is applied when the workflow is triggered by a schedule event
+          # type=ref,event=branch means that a tag is applied when the workflow is triggered by a push to a branch
+          # type=ref,event=tag means that a tag is applied when the workflow is triggered by a push to a tag
+        tags: |
+          type=schedule,suffix=-${{ inputs.platform }},enable=${{ github.event_name == 'schedule' }}
+          type=ref,event=branch,suffix=-${{ inputs.platform }},enable=${{ github.event_name != 'schedule' }}
+          type=ref,event=tag,suffix=-${{ inputs.platform }}
+    -
+      name: Create manifest list and push to repository
+      working-directory: /tmp/digests/${{ inputs.registry }}/${{ inputs.platform }}
+      # When using composite actions, you have to specify the shell. As you
+      # don’t specify a runner type in composite actions, you need to specify
+      # the shell instead for each action.
+      shell: bash
+      run: |
+        docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf '${{ inputs.imagename }}@sha256:%s ' *)
+    -
+      name: Inspect image
+      shell: bash
+      run: |
+        docker buildx imagetools inspect ${{ inputs.imagename }}:${{ steps.meta_docker.outputs.version }}

.github/workflows/ftl-build.yml

@@ -1,12 +1,15 @@
 name: ftl-build builds
 on:
-  pull_request:
-    paths:
-      - 'ftl-build/**'
-      - '.github/workflows/ftl-build.yml'
   push:
+    # Run on all branches and tags, but only if the ftl-build directory or the
+    # workflow itself changed. This is to avoid running the workflow on
+    # unrelated changes.
+    # Note that including the branch filter is necessary as otherwise (paths and tags alone),
+    # the workflow would not run on pushes to branches that do not have a tag sticked to them.
     tags:
       - "**"
+    branches:
+      - "**"
     paths:
       - 'ftl-build/**'
       - '.github/workflows/ftl-build.yml'
@@ -15,83 +18,180 @@ on:
     # 1:30am UTC every Sunday, has no particular significance
     - cron: "30 1 * * 0"
 
+env:
+  DOCKER_REGISTRY_IMAGE: ${{ secrets.DOCKERHUB_NAMESPACE }}/ftl-build
+  GITHUB_REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/ftl-build
+
 jobs:
-  build:
+  build-and-test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        ARCH: [aarch64, armv4t, armv5te, armv6hf, armv7hf, armv8a, x86_32, x86_64, x86_64-musl, riscv64]
+        include:
+          - platform: linux/amd64
+            container: alpine:3.18
+          - platform: linux/386
+            container: alpine:3.18
+          - platform: linux/arm/v5
+            container: debian:stretch-slim
+          - platform: linux/arm/v6
+            container: debian:bullseye-slim
+          - platform: linux/arm/v6
+            container: alpine:3.18
+          - platform: linux/arm/v7
+            container: alpine:3.18
+          - platform: linux/arm64/v8
+            container: alpine:3.18
+          - platform: linux/riscv64
+            container: alpine:edge
+    env:
+      context: ${{ startsWith(matrix.container, 'alpine') && 'alpine' || 'debian' }}
     steps:
       -
         name: Checkout Repo
         uses: actions/checkout@v3
       -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      -
-        name: Docker meta
+        name: Docker meta (Docker Hub and GitHub Container Registry)
         id: meta
         uses: docker/metadata-action@v4
         with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           images: |
-            ${{ secrets.DOCKERHUB_NAMESPACE }}/ftl-build,enable=${{ github.event_name != 'pull_request' }}
-            ghcr.io/${{ github.repository_owner }}/ftl-build,enable=${{ github.event_name != 'pull_request' }}
-            foo/bar,enable=${{ github.event_name == 'pull_request' }}
-          flavor: |
-            latest=false
+            ${{ env.DOCKER_REGISTRY_IMAGE }},enable=${{ github.event_name != 'workflow_dispatch' }}
+            ${{ env.GITHUB_REGISTRY_IMAGE }},enable=${{ github.event_name != 'workflow_dispatch' }}
+            foo/bar,enable=${{ github.event_name == 'workflow_dispatch' }}
           tags: |
-            type=ref,event=tag,suffix=-${{ matrix.ARCH }}
-            type=ref,event=branch,suffix=-${{ matrix.ARCH }},enable=${{ github.event_name != 'schedule' }}
-            type=ref,event=pr,suffix=-${{ matrix.ARCH }}
-            type=raw,value=${{matrix.ARCH}},enable=${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' }}
-      -
-        name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+            type=ref,event=branch,enable=${{ github.event_name != 'schedule' }}
+      -
+        name: Login to DockerHub and GitHub Container Registry
+        uses: ./.github/actions/login-repo
         with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
+          docker_username: ${{ secrets.DOCKERHUB_USER }}
+          docker_password: ${{ secrets.DOCKERHUB_PASS }}
+          ghcr_username: ${{ github.repository_owner }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Login to GitHub Container Registry
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        # Add support for more platforms with QEMU (optional)
+        # https://github.com/docker/setup-qemu-action
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
         with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          platforms: all
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       -
-        name: Build
+        name: Build container and test-compile FTL
         uses: docker/build-push-action@v4
         with:
-          context: ftl-build/${{ matrix.ARCH }}/.
+          context: ftl-build/${{ env.context }}
+          platforms: ${{ matrix.platform }}
           push: false
           target: tester
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          # Caching disabled for now...
-          # cache-from: type=gha,scope=${{ matrix.ARCH }}
-          # cache-to: type=gha,scope=${{ matrix.ARCH }},mode=max
+          build-args: |
+            CONTAINER=${{ matrix.container }}
       -
-        name: Build (all-in)
-        if: matrix.ARCH == 'x86_64'
+        name: Push builder target and push by digest (Docker Hub)
+        id: build_docker
         uses: docker/build-push-action@v4
         with:
-          context: ftl-build/${{ matrix.ARCH }}/.
-          push: false
-          target: tester-all-in
-          tags: ${{ steps.meta.outputs.tags }}
+          context: ftl-build/${{ env.context }}
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'workflow_dispatch' }}
+          target: builder
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            CONTAINER=${{ matrix.container }}
+          outputs: |
+            type=image,name=${{ env.DOCKER_REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
       -
-        name: Push
-        if: github.event_name != 'pull_request'
+        name: Push builder target and push by digest (GitHub Container Registry)
+        id: build_ghcr
         uses: docker/build-push-action@v4
         with:
-          context: ftl-build/${{ matrix.ARCH }}/.
-          push: true
+          context: ftl-build/${{ env.context }}
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'workflow_dispatch' }}
           target: builder
-          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          # Caching disabled for now...
-          # cache-from: type=gha,scope=${{ matrix.ARCH }}
-          # cache-to: type=gha,scope=${{ matrix.ARCH }},mode=max
+          build-args: |
+            CONTAINER=${{ matrix.container }}
+          outputs: |
+            type=image,name=${{ env.GITHUB_REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+      -
+        name: Export digests
+        run: |
+          mkdir -p /tmp/digests/dockerhub/${{ env.context }}
+          mkdir -p /tmp/digests/ghcr/${{ env.context }}
+          digest_docker="${{ steps.build_docker.outputs.digest }}"
+          touch "/tmp/digests/dockerhub/${{ env.context }}/${digest_docker#sha256:}"
+          digest_ghcr="${{ steps.build_ghcr.outputs.digest }}"
+          touch "/tmp/digests/ghcr/${{ env.context }}/${digest_ghcr#sha256:}"
+      -
+        name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  # Merge all the digests into a single file
+  # If we would push immediately above, the individual runners would overwrite each other's images
+  # https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+  merge-and-deploy:
+    runs-on: ubuntu-latest
+    needs:
+      - build-and-test
+    steps:
+      -
+        name: Checkout Repo
+        uses: actions/checkout@v3
+      -
+        name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to DockerHub and GitHub Container Registry
+        uses: ./.github/actions/login-repo
+        with:
+          docker_username: ${{ secrets.DOCKERHUB_USER }}
+          docker_password: ${{ secrets.DOCKERHUB_PASS }}
+          ghcr_username: ${{ github.repository_owner }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Collect and push (Alpine, Docker Hub)
+        uses: ./.github/actions/merge-and-push
+        with:
+          imagename: ${{ env.DOCKER_REGISTRY_IMAGE }}
+          platform: alpine
+          registry: dockerhub
+      -
+        name: Collect and push (Debian, Docker Hub)
+        uses: ./.github/actions/merge-and-push
+        with:
+          imagename: ${{ env.DOCKER_REGISTRY_IMAGE }}
+          platform: debian
+          registry: dockerhub
+      -
+        name: Collect and push (Alpine, GitHub Container Registry)
+        uses: ./.github/actions/merge-and-push
+        with:
+          imagename: ${{ env.GITHUB_REGISTRY_IMAGE }}
+          platform: alpine
+          registry: ghcr
+      -
+        name: Collect and push (Debian, GitHub Container Registry)
+        uses: ./.github/actions/merge-and-push
+        with:
+          imagename: ${{ env.GITHUB_REGISTRY_IMAGE }}
+          platform: debian
+          registry: ghcr