feat: Enable multi-arch builds

AMD64 and ARM64 for now. The Tyk plugin depends on the tykio/tyk-plugin-compiler image which does not support any other architecture than AMD64 for now. See TykTechnologies/tyk-build-env#4 ARMv7 was tried but Kong and Tyk do not support it and it may introduce issues due to its 32-bit architecture that we don't want to have to support. Signed-off-by: Felix Kaechele <[email protected]>
openclarity · May 10, 2023 · 26c2e1e · 26c2e1e
1 parent 26129dd
commit 26c2e1e
Show file tree

Hide file tree

Showing 8 changed files with 154 additions and 34 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -26,6 +26,13 @@ jobs:
     - name: Verify spec aggregation and code generation
       run: make api3-verify
 
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+
     - name: Check licenses
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -34,8 +41,66 @@ jobs:
     - name: Run verification
       run: make check
 
-    - name: Build Docker image
-      run: make docker
+    - name: Get current timestamp
+      id: timestamp
+      run: echo "::set-output name=timestamp::$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+
+    - name: Build APIClarity
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        platforms: linux/amd64,linux/arm64
+        tags: ghcr.io/apiclarity/apiclarity:${{ github.sha }}
+        file: Dockerfile
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
+        build-args: |
+          VERSION=${{ github.sha }}
+          BUILD_TIMESTAMP=${{ steps.timestamp.outputs.timestamp }}
+          COMMIT_HASH=${{ github.sha }}
+
+    - name: Build Kong plugin
+      uses: docker/build-push-action@v2
+      with:
+        context: plugins
+        platforms: linux/amd64,linux/arm64
+        tags: ghcr.io/apiclarity/kong-plugin:${{ github.sha }}
+        file: plugins/Dockerfile.kong
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
+
+    - name: Fix Tyk Plugin API dep
+      id: tyk-dep
+      run: |
+        cd plugins/gateway/tyk/v3.2.2
+        go mod edit -replace github.com/apiclarity/apiclarity/plugins/[email protected]=./../api
+        go mod edit -replace github.com/apiclarity/apiclarity/plugins/[email protected]=./../common
+
+    - name: Build Tyk plugin
+      uses: docker/build-push-action@v2
+      with:
+        context: plugins
+        platforms: linux/amd64 # tykio/tyk-plugin-compiler does not currently support other architectures
+        tags: ghcr.io/apiclarity/tyk-plugin-v3.2.2:${{ github.sha }}
+        file: plugins/Dockerfile.tyk.v3.2.2
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
+
+    - name: Undo Tyk Plugin API dep change
+      id: undo-tyk-dep
+      run: git checkout -- plugins/gateway/tyk/v3.2.2/go.mod
+
+    - name: Build passive taper
+      uses: docker/build-push-action@v2
+      with:
+        context: plugins
+        platforms: linux/amd64,linux/arm64
+        tags: ghcr.io/apiclarity/passive-taper:${{ github.sha }}
+        file: plugins/Dockerfile.taper
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
+        build-args: |
+          VERSION=${{ github.sha }}
 
   lint_chart:
     name: Lint Helm Chart

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -17,6 +17,9 @@ jobs:
         with:
           submodules: true
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
@@ -43,6 +46,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/apiclarity:latest
           file: Dockerfile
           push: true
@@ -57,6 +61,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/kong-plugin:latest
           file: plugins/Dockerfile.kong
           push: true
@@ -74,6 +79,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64 # tykio/tyk-plugin-compiler does not currently support other architectures
           tags: ghcr.io/openclarity/tyk-plugin-v3.2.2:latest
           file: plugins/Dockerfile.tyk.v3.2.2
           push: true
@@ -84,6 +90,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/passive-taper:latest
           file: plugins/Dockerfile.taper
           push: true

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -82,6 +82,9 @@ jobs:
           git tag --annotate --message "Tag for release $tag" "$tag"
           git push origin "refs/tags/$tag"
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
@@ -108,6 +111,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/apiclarity:${{ github.event.inputs.version }}
           file: Dockerfile
           push: true
@@ -122,6 +126,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/kong-plugin:${{ github.event.inputs.version }}
           file: plugins/Dockerfile.kong
           push: true
@@ -139,6 +144,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64 # tykio/tyk-plugin-compiler does not currently support other architectures
           tags: ghcr.io/openclarity/tyk-plugin-v3.2.2:${{ github.event.inputs.version }}
           file: plugins/Dockerfile.tyk.v3.2.2
           push: true
@@ -153,6 +159,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: plugins
+          platforms: linux/amd64,linux/arm64
           tags: ghcr.io/openclarity/passive-taper:${{ github.event.inputs.version }}
           file: plugins/Dockerfile.taper
           push: true

diff --git a/Dockerfile b/Dockerfile
@@ -1,31 +1,41 @@
-FROM node:14-slim AS site-build
+## Build Frontend
+FROM --platform=$BUILDPLATFORM node:14-slim AS site-build
 
 WORKDIR /app/ui-build
 
+# Cache optimization: Avoid npm install unless package.json changed
+COPY ui/package-lock.json ui/package.json ./
+RUN npm ci
+
 COPY ui .
-RUN npm i
 RUN npm run build
 
-FROM golang:1.17.9-alpine AS builder
+## Build Backend
+# Cross-compilation tools
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
+
+FROM --platform=$BUILDPLATFORM golang:1.17.9-alpine AS builder
 
-RUN apk add --update --no-cache gcc g++
+# Copy cross-compilation tools
+COPY --from=xx / /
 
 WORKDIR /build
 COPY api ./api
 COPY api3 ./api3
 COPY plugins/api ./plugins/api
 
+# Cache optimization: Avoid go module downloads unless go.mod/go.sum changed
 WORKDIR /build/backend
 COPY backend/go.* ./
 RUN go mod download
 
-ARG VERSION
-ARG BUILD_TIMESTAMP
-ARG COMMIT_HASH
+ARG BUILD_TIMESTAMP COMMIT_HASH VERSION TARGETOS TARGETARCH
 
 # Copy and build backend code
 COPY backend .
-RUN go build -ldflags="-s -w \
+ARG TARGETPLATFORM
+ENV CGO_ENABLED=0
+RUN xx-go build -ldflags="-s -w \
      -X 'github.com/openclarity/apiclarity/backend/pkg/version.Version=${VERSION}' \
      -X 'github.com/openclarity/apiclarity/backend/pkg/version.CommitHash=${COMMIT_HASH}' \
      -X 'github.com/openclarity/apiclarity/backend/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}'" -o backend ./cmd/backend/main.go

diff --git a/Makefile b/Makefile
@@ -67,7 +67,8 @@ docker:	docker-backend docker-plugins
 docker-backend: ## Build Docker image
 	@(echo "Building backend docker image ..." )
 	@(cd backend)
-	docker build --build-arg VERSION=${VERSION} \
+	docker buildx build --load \
+		--build-arg VERSION=${VERSION} \
 		--build-arg BUILD_TIMESTAMP=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") \
 		--build-arg COMMIT_HASH=$(shell git rev-parse HEAD) \
 		-t ${DOCKER_IMAGE}:${DOCKER_TAG} .

diff --git a/plugins/Dockerfile.kong b/plugins/Dockerfile.kong
@@ -1,28 +1,39 @@
-FROM golang:1.17.8-alpine3.14 AS builder
+## Common build steps for plugins
+# Cross-compilation tools
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
 
-## Note: this Dockerfile will only be built from the plugins context
+FROM --platform=$BUILDPLATFORM golang:1.17.8-alpine3.14 AS builder
 
-RUN apk add --update --no-cache gcc g++
+# Copy cross-compilation tools
+COPY --from=xx / /
 
-WORKDIR /plugins
-COPY api ./api
+# To optimize caching and multi-architecture we try to get in as much upfront
+# work as possible before branching out using our actual source
+WORKDIR /plugins/api
+COPY api/go.* ./
+RUN go mod download
 
 WORKDIR /plugins/common
-
 COPY common/go.* ./
 RUN go mod download
 
-COPY common ./
+WORKDIR /plugins
+COPY api ./api
+COPY common ./common
 
+## Kong plugin build steps
 WORKDIR /plugins/gateway/kong
-COPY gateway/kong/go.* ./
 
+# Cache optimization: Avoid go module downloads unless go.mod/go.sum changed
+COPY gateway/kong/go.* ./
 RUN go mod download
 
 COPY gateway/kong .
 
 # Build the plugin.
-RUN go build -o bin/kong-plugin plugin.go
+ARG TARGETPLATFORM
+ENV CGO_ENABLED=0
+RUN xx-go build -o bin/kong-plugin plugin.go
 
 FROM busybox
 COPY --from=builder ["/plugins/gateway/kong/bin/kong-plugin", "/kong-plugin"]
diff --git a/plugins/Dockerfile.taper b/plugins/Dockerfile.taper
@@ -1,37 +1,56 @@
-FROM golang:1.16.6-alpine AS builder
+## Common build steps for plugins
+# Cross-compilation tools
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
 
-# Set necessary environment variables needed for our image.
-ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
+FROM --platform=$BUILDPLATFORM golang:1.16.6-alpine AS builder
 
-RUN apk add libpcap-dev gcc g++ make bash
+# Copy cross-compilation tools
+COPY --from=xx / /
 
-WORKDIR /plugins
+# To optimize caching and multi-architecture we try to get in as much upfront
+# work as possible before branching out using our actual source
+WORKDIR /plugins/api
+COPY api/go.* ./
+RUN go mod download
 
+WORKDIR /plugins/common
+COPY common/go.* ./
+RUN go mod download
+
+WORKDIR /plugins
 COPY api ./api
 COPY common ./common
 
-WORKDIR /plugins/taper
+## Taper plugin build steps
+# Cache optimization: Avoid go module downloads unless go.mod/go.sum changed
+WORKDIR /plugins/taper/extensions/http
+COPY taper/extensions/http/go.* ./
+RUN go mod download
 
+WORKDIR /plugins/taper
 COPY taper/go.* ./
 RUN go mod download
 
-ARG VERSION
+ARG TARGETPLATFORM VERSION
 COPY taper .
-RUN go build -ldflags="-s -w \
+RUN apk add --no-cache clang lld
+RUN xx-apk add --no-cache binutils-gold gcc libpcap-dev musl-dev
+ENV CGO_ENABLED=1
+RUN export QEMU_LD_PREFIX=/$(xx-info) && xx-go build -ldflags="-s -w \
      -X 'github.com/openclarity/apiclarity/plugins/taper/version.Version=${VERSION}'" -o agent .
 
 WORKDIR /plugins/taper/extensions/http
 
 COPY taper/extensions/http .
-RUN go build -buildmode=plugin -o ../http.so .
+RUN export QEMU_LD_PREFIX=/$(xx-info) && xx-go build -buildmode=plugin -o ../http.so .
 
 FROM alpine:3.14
 
-RUN apk add bash libpcap-dev tcpdump
+RUN apk add --no-cache bash libpcap tcpdump
 WORKDIR /app
 
 # Copy binary and config files from /build to root folder of scratch container.
 COPY --from=builder ["/plugins/taper/agent", "."]
 COPY --from=builder ["/plugins/taper/extensions/http.so", "extensions/http.so"]
 
-ENTRYPOINT ["/app/agent"]
+ENTRYPOINT ["/app/agent"]
diff --git a/plugins/Makefile b/plugins/Makefile
@@ -10,19 +10,19 @@ docker: docker-kong docker-tyk docker-taper docker-otelcollector
 
 .PHONY: docker-kong
 docker-kong:
-	@DOCKER_BUILDKIT=1 docker build -t ${DOCKER_REGISTRY}/kong-plugin:${DOCKER_TAG} --file Dockerfile.kong .
+	@DOCKER_BUILDKIT=1 docker buildx build --load -t ${DOCKER_REGISTRY}/kong-plugin:${DOCKER_TAG} --file Dockerfile.kong .
 
 .PHONY: docker-taper
 docker-taper:
-	@DOCKER_BUILDKIT=1 docker build --build-arg VERSION=${VERSION} -t ${DOCKER_REGISTRY}/passive-taper:${DOCKER_TAG} --file Dockerfile.taper .
+	@DOCKER_BUILDKIT=1 docker buildx build --load --build-arg VERSION=${VERSION} -t ${DOCKER_REGISTRY}/passive-taper:${DOCKER_TAG} --file Dockerfile.taper .
 
 .PHONY: docker-tyk
 docker-tyk:
 	@(cd gateway/tyk/${TYK_VERSION} && \
 	go mod edit -replace github.com/openclarity/apiclarity/plugins/[email protected]=./../api && \
 	go mod edit -replace github.com/openclarity/apiclarity/plugins/[email protected]=./../common && \
 	 cd -)
-	@DOCKER_BUILDKIT=1 docker build -t ${DOCKER_REGISTRY}/tyk-plugin-${TYK_VERSION}:${DOCKER_TAG} --file Dockerfile.tyk.${TYK_VERSION} .
+	@DOCKER_BUILDKIT=1 docker buildx build --load -t ${DOCKER_REGISTRY}/tyk-plugin-${TYK_VERSION}:${DOCKER_TAG} --file Dockerfile.tyk.${TYK_VERSION} .
 	@(cd gateway/tyk/${TYK_VERSION} && \
 	go mod edit -replace github.com/openclarity/apiclarity/plugins/[email protected]=./../../../api && \
 	go mod edit -replace github.com/openclarity/apiclarity/plugins/[email protected]=./../../../common && \