Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

build: refactor install_ffmpeg.sh, add macos --> linux support #2886

Merged
merged 1 commit into from
Oct 25, 2023
Merged

build: refactor install_ffmpeg.sh, add macos --> linux support #2886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 56 additions & 49 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,43 +14,52 @@ concurrency:

jobs:
linux-build:
name: Build binaries for ${{ matrix.platform.name }}-${{ matrix.hardware.type }}-${{ matrix.arch }}
runs-on: ${{ matrix.platform.runner }}
name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.type }}-${{ matrix.target.GOARCH }}
runs-on: ${{ matrix.target.runner }}
container:
image: ${{ matrix.hardware.container }}
image: ${{ matrix.target.container }}
env:
DEBIAN_FRONTEND: noninteractive
strategy:
fail-fast: false
matrix:
arch:
- arm64
- amd64
platform:
- name: linux
target:
- GOOS: linux
GOARCH: amd64
runner: ubuntu-20.04
- name: windows
container: ubuntu:20.04
type: cpu

- GOOS: linux
GOARCH: arm64
runner: ubuntu-20.04
hardware:
- type: cpu
container: ubuntu:20.04
- type: gpu
type: cpu

- GOOS: linux
GOARCH: amd64
runner: ubuntu-20.04
container: livepeerci/cuda:11.7.1-cudnn8-devel-ubuntu20.04
exclude:
- platform:
name: windows
arch: arm64
- platform:
name: windows
hardware:
type: gpu
type: gpu

- GOOS: linux
GOARCH: arm64
runner: ubuntu-20.04
container: livepeerci/cuda:11.7.1-cudnn8-devel-ubuntu20.04
type: gpu

- GOOS: windows
GOARCH: amd64
runner: ubuntu-20.04
container: ubuntu:22.04
type: cpu

steps:
- name: Setup ubuntu container
run: |
apt update
apt install -yqq build-essential make software-properties-common
add-apt-repository -y ppa:git-core/ppa
add-apt-repository -y ppa:git-core/candidate
apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g yasm
- name: Check out code
Expand All @@ -74,39 +83,37 @@ jobs:
uses: actions/cache@v3
with:
path: /home/runner/compiled
key: ${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.hardware.type }}-${{ matrix.arch }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.hardware.type }}-${{ matrix.arch }}-ffmpeg-
key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-${{ matrix.target.type }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}

- name: Set build environment
run: |
echo "GOARCH=${{ matrix.arch }}" >> $GITHUB_ENV
echo "GOOS=${{ matrix.platform.name }}" >> $GITHUB_ENV
echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV
echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV
echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV
- name: Set GPU build environment
if: matrix.hardware.type == 'gpu'
if: matrix.target.type == 'gpu'
run: |
echo "CPATH=/usr/local/cuda_${{ matrix.arch }}/include" >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/local/cuda_${{ matrix.arch }}/lib64" >> $GITHUB_ENV
echo "CGO_LDFLAGS=-L/usr/local/cuda_${{ matrix.arch }}/lib64" >> $GITHUB_ENV
echo "CPATH=/usr/local/cuda_${{ matrix.target.GOARCH }}/include" >> $GITHUB_ENV
echo "LIBRARY_PATH=/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV
echo "CGO_LDFLAGS=-L/usr/local/cuda_${{ matrix.target.GOARCH }}/lib64" >> $GITHUB_ENV
echo "RELEASE_TAG=gpu" >> $GITHUB_ENV
- name: Install dependencies
run: |
apt update \
&& apt install -yqq software-properties-common curl apt-transport-https lsb-release \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \
&& apt update \
&& apt -yqq install \
clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python \
gcc-multilib gcc-mingw-w64 libgcc-9-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64
nasm clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python3 \
gcc-mingw-w64 libgcc-9-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \
&& update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30
- name: Install go modules
if: steps.go.outputs.cache-hit != 'true'
Expand Down Expand Up @@ -142,16 +149,18 @@ jobs:
run: curl -X POST https://holy-bread-207a.livepeer.workers.dev

macos-build:
name: Build binaries for ${{ matrix.platform.name }}-${{ matrix.arch }}
runs-on: ${{ matrix.platform.runner }}
name: Build binaries for ${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}
runs-on: ${{ matrix.target.runner }}
strategy:
fail-fast: false
matrix:
arch:
- arm64
- amd64
platform:
- name: darwin
target:
- GOOS: darwin
GOARCH: amd64
runner: macos-latest

- GOOS: darwin
GOARCH: arm64
runner: macos-latest

steps:
Expand All @@ -173,18 +182,16 @@ jobs:

- name: Set build environment
run: |
echo "GOARCH=${{ matrix.arch }}" >> $GITHUB_ENV
echo "GOOS=${{ matrix.platform.name }}" >> $GITHUB_ENV
echo "GOARCH=${{ matrix.target.GOARCH }}" >> $GITHUB_ENV
echo "GOOS=${{ matrix.target.GOOS }}" >> $GITHUB_ENV
echo "GO_BUILD_DIR=lp-builds/" >> $GITHUB_ENV
- name: Cache ffmpeg
id: cache-ffmpeg
uses: actions/cache@v3
with:
path: ~/compiled
key: ${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-ffmpeg-
key: ${{ runner.os }}-${{ matrix.target.GOOS }}-${{ matrix.target.GOARCH }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}

- name: Install dependencies
run: brew install coreutils pkg-config
Expand Down Expand Up @@ -212,7 +219,7 @@ jobs:
regex: '^(master|main|v[0-9]+\.\d+\.\d+)$'

- name: Codesign and notarize binaries
if: steps.match-tag.outputs.match != '' && matrix.platform.name == 'darwin'
if: steps.match-tag.outputs.match != '' && matrix.target.GOOS == 'darwin'
uses: livepeer/action-gh-codesign-apple@latest
with:
developer-certificate-id: ${{ secrets.CI_MACOS_CERTIFICATE_ID }}
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,13 @@ jobs:
sudo apt update \
&& sudo apt install -yqq software-properties-common curl apt-transport-https lsb-release \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - \
&& sudo add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& sudo add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \
&& sudo apt update \
&& sudo apt -yqq install clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python
&& sudo apt -yqq install clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& sudo update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \
&& sudo update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30
- name: Install go modules
# if: steps.go.outputs.cache-hit != 'true'
Expand Down
61 changes: 48 additions & 13 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,30 +22,65 @@ cgo_cflags :=
cgo_ldflags :=
cc :=

uname_s := $(shell uname -s)
ifeq ($(uname_s),Darwin)
# Build platform flags
BUILDOS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
BUILDARCH ?= $(shell uname -m | tr '[:upper:]' '[:lower:]')
ifeq ($(BUILDARCH),aarch64)
BUILDARCH=arm64
endif
ifeq ($(BUILDARCH),x86_64)
BUILDARCH=amd64
endif

# Override these (not BUILDOS/BUILDARCH) for cross-compilation
export GOOS ?= $(BUILDOS)
export GOARCH ?= $(BUILDARCH)

# Cross-compilation section. Currently supported:
# darwin amd64 --> darwin arm64
# darwin arm64 --> linux arm64
# linux amd64 --> linux arm64
# linux amd64 --> windows amd64

ifeq ($(BUILDOS),darwin)
ifeq ($(GOOS),darwin)
cgo_ldflags += -framework CoreFoundation -framework Security
ifeq ($(GOARCH),arm64)
cgo_cflags += --target=arm64-apple-macos11
cgo_ldflags += --target=arm64-apple-macos11
ifeq ($(BUILDARCH),amd64)
ifeq ($(GOARCH),arm64)
cgo_cflags += --target=arm64-apple-macos11
cgo_ldflags += --target=arm64-apple-macos11
endif
endif
endif
endif

ifeq ($(uname_s),Linux)
ifeq ($(GOARCH),arm64)
cgo_cflags += --target=aarch64-linux-gnu
cgo_ldflags += --target=aarch64-linux-gnu
cc = clang --sysroot=/usr/aarch64-linux-gnu
ifeq ($(GOOS),linux)
ifeq ($(GOARCH),arm64)
LLVM_PATH ?= /opt/homebrew/opt/llvm/bin
SYSROOT ?= /tmp/sysroot-aarch64-linux-gnu
cgo_cflags += --target=aarch64-linux-gnu -Wno-error=unused-command-line-argument -fuse-ld=$(LLVM_PATH)/ld.lld
cgo_ldflags += --target=aarch64-linux-gnu
cc = $(LLVM_PATH)/clang -fuse-ld=$(LLVM_PATH)/ld.lld --sysroot=$(SYSROOT)
endif
endif
endif

pkg_config_libdir :=
ifeq ($(uname_s),Linux)
ifeq ($(BUILDOS),linux)
ifeq ($(BUILDARCH),amd64)
ifeq ($(GOARCH),arm64)
ifeq ($(GOOS),linux)
cgo_cflags += --target=aarch64-linux-gnu
cgo_ldflags += --target=aarch64-linux-gnu
cc = clang
endif
endif
endif

ifeq ($(GOOS),windows)
cc = x86_64-w64-mingw32-gcc
endif
endif


.PHONY: livepeer livepeer_bench livepeer_cli livepeer_router docker

livepeer:
Expand Down
10 changes: 5 additions & 5 deletions docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ RUN apt update \
&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& add-apt-repository "deb [arch=${BUILDARCH}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb [arch=${BUILDARCH}] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& add-apt-repository "deb [arch=${BUILDARCH}] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \
&& apt update \
&& apt -yqq install clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python docker-ce-cli pciutils gcc-multilib libgcc-8-dev-arm64-cross gcc-mingw-w64-x86-64
&& apt -yqq install clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python docker-ce-cli pciutils gcc-multilib libgcc-8-dev-arm64-cross gcc-mingw-w64-x86-64

RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \
&& update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30

RUN GRPC_HEALTH_PROBE_VERSION=v0.3.6 \
&& curl -fsSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} -o /usr/bin/grpc_health_probe \
Expand Down
Loading
Loading