build broken for bundle target for ARM architectures when USE_OPENSSL=ON

[qrkourier]

I tried to build the arm64 binary and ran into an issue with USE_OPENSSL=ON. I recognize that we're using mbedtls in GitHub CI, and the package builds are amd64 only, and so I'm wondering if building for ARM architectures with openssl is supported at all. I've attached the build log: arm64-openssl.log

Here's the incantation I attempted to build for ARM64.

cmake \
                    -DCMAKE_BUILD_TYPE=Release \
                    -DCMAKE_TOOLCHAIN_FILE=${REPO_DIR}/toolchains/Linux-arm64.cmake \
                    -DUSE_OPENSSL=ON \
                    -S ${REPO_DIR} \
                    -B ${CMAKE_BUILD_DIR} \
&& cmake \
                    --build ${CMAKE_BUILD_DIR} \
                    --target bundle \
                    --verbose;

[ekoby]

architecture specific includes are in /usr/include/<arch-id>-linux-gnu/openssl/
you need to apt install libssl-dev:arm64

see this https://askubuntu.com/questions/430705/how-to-use-apt-get-to-download-multi-arch-library

[qrkourier]

@ekoby Thanks for that guidance. The build progresses further with that dependency satisfied. I addressed the issue by running:

dpkg --add-architecture arm64
dpkg --add-architecture armhf
apt update
apt -y install  libssl-dev:arm64 \
                libssl-dev:armhf

The next error is right near the end of the build.

arm64

[100%] Linking C executable ziti-edge-tunnel
cd /mnt/build-arm64/programs/ziti-edge-tunnel && /usr/bin/cmake -E cmake_link_script CMakeFiles/ziti-edge-tunnel.dir/link.txt --verbose=1
/usr/bin/aarch64-linux-gnu-gcc -O3 -DNDEBUG CMakeFiles/ziti-edge-tunnel.dir/ziti-edge-tunnel.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/tun.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/resolvers.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance.c.o CMakeFiles/ziti-edge-tunnel.dir/config-utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance-config.c.o -o ziti-edge-tunnel   -L/mnt/build-arm64/lib  -Wl,-rpath,/mnt/build-arm64/lib:/usr/lib/x86_64-linux-gnu: ../../_deps/ziti-sdk-c-build/library/libziti.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../lib/ziti-tunnel-cbs/libziti-tunnel-cbs-c.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../_deps/ziti-sdk-c-build/library/libziti.a ../../_deps/uv-mbed-build/libuv_mbed.a /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/x86_64-linux-gnu/libcrypto.so ../../_deps/libsodium-build/lib/libsodium.a -latomic -lm ../../lib/ziti-tunnel/liblwipcore.a ../../_deps/libuv-build/libuv_a.a -lpthread -ldl -lrt -lresolv 
/usr/lib/gcc-cross/aarch64-linux-gnu/10/../../../../aarch64-linux-gnu/bin/ld: /usr/lib/x86_64-linux-gnu/libssl.so: error adding symbols: file in wrong format
collect2: error: ld returned 1 exit status

arm

[100%] Linking C executable ziti-edge-tunnel
cd /mnt/build-arm/programs/ziti-edge-tunnel && /usr/bin/cmake -E cmake_link_script CMakeFiles/ziti-edge-tunnel.dir/link.txt --verbose=1
/usr/bin/arm-linux-gnueabihf-gcc -O3 -DNDEBUG CMakeFiles/ziti-edge-tunnel.dir/ziti-edge-tunnel.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/tun.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/resolvers.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance.c.o CMakeFiles/ziti-edge-tunnel.dir/config-utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance-config.c.o -o ziti-edge-tunnel   -L/mnt/build-arm/lib  -Wl,-rpath,/mnt/build-arm/lib:/usr/lib/x86_64-linux-gnu: ../../_deps/ziti-sdk-c-build/library/libziti.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../lib/ziti-tunnel-cbs/libziti-tunnel-cbs-c.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../_deps/ziti-sdk-c-build/library/libziti.a ../../_deps/uv-mbed-build/libuv_mbed.a /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/x86_64-linux-gnu/libcrypto.so ../../_deps/libsodium-build/lib/libsodium.a -latomic -lm ../../lib/ziti-tunnel/liblwipcore.a ../../_deps/libuv-build/libuv_a.a -lpthread -ldl -lrt -lresolv 
/usr/lib/x86_64-linux-gnu/libssl.so: file not recognized: file format not recognized
collect2: error: ld returned 1 exit status

The path to libssl.so in both cases includes "x86_64" which is the wrong architecture for the crossbuild.

[qrkourier]

I can still reproduce this build error.

arm

# verbatim configure and build commands
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=/mnt/docker/../toolchains/Linux-arm.cmake -DUSE_OPENSSL=ON -S /mnt/docker/.. -B /mnt/docker/../build-arm     
cmake --build /mnt/docker/../build-arm --target package --verbose

after configuring cmake the second cmake build command gets:

cd /mnt/build-arm/programs/ziti-edge-tunnel && /usr/local/bin/cmake -E cmake_link_script CMakeFiles/ziti-edge-tunnel.dir/link.txt --verbose=1
/usr/bin/arm-linux-gnueabihf-gcc -O3 -DNDEBUG CMakeFiles/ziti-edge-tunnel.dir/ziti-edge-tunnel.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/tun.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/resolvers.c.o CMakeFiles/ziti-edge-tunnel.dir/netif_driver/linux/utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance.c.o CMakeFiles/ziti-edge-tunnel.dir/config-utils.c.o CMakeFiles/ziti-edge-tunnel.dir/instance-config.c.o -o ziti-edge-tunnel   -L/mnt/build-arm/lib  -Wl,-rpath,/mnt/build-arm/lib:/usr/lib/x86_64-linux-gnu: ../../_deps/ziti-sdk-c-build/library/libziti.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../lib/ziti-tunnel-cbs/libziti-tunnel-cbs-c.a ../../lib/ziti-tunnel/libziti-tunnel-sdk-c.a ../../_deps/ziti-sdk-c-build/library/libziti.a ../../_deps/uv-mbed-build/libuv_mbed.a /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/x86_64-linux-gnu/libcrypto.so ../../_deps/libsodium-build/lib/libsodium.a -latomic -lm ../../lib/ziti-tunnel/liblwipcore.a ../../_deps/libuv-build/libuv_a.a -lpthread -ldl -lrt -lresolv 
/usr/lib/x86_64-linux-gnu/libssl.so: file not recognized: file format not recognized
collect2: error: ld returned 1 exit status

[qrkourier]

Is I change the config param to USE_OPENSSL=no then the Mbed-TLS build for arm succeeds.

❯ file build-arm/programs/ziti-edge-tunnel/ziti-edge-tunnel
build-arm/programs/ziti-edge-tunnel/ziti-edge-tunnel: ELF 32-bit LSB pie executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, BuildID[sha1]=e5734a9662808ddf4238799c0bf558f0e28d1b39, for GNU/Linux 3.2.0, with debug_info, not stripped

There must be an issue with the way I'm building for OpenSSL.

[qrkourier]

Reproduce the failed executable build for arch arm w/ OpenSSL with these steps run in top-level repo dir on branch issue-449-crossbuild-arm-package:

docker build --file ./docker/Dockerfile.linux-build -t ziti-edge-tunnel-builder .
docker run --name ziti-edge-tunnel-builder --rm --volume ${PWD}:/mnt ziti-edge-tunnel-builder --openssl arm

[qrkourier]

I added the crossbuild-essential-armhf package to the debian:stable-slim build env and I get the same wrong arch libssl.so error reported earlier, but I finally noticed this clue in the build spew:

-- summary of build options:                                                                                                                                            
    Install prefix:  /opt/openziti/ziti-sdk-0.30.4                                                                                                                      
    Target system:   Linux                                                                                                                                              
    Compiler:                                                                                                                                                           
      C compiler:    /usr/bin/arm-linux-gnueabihf-gcc (GNU)                                                                                                             
      CFLAGS:                                                                                                                                                           
                                                                                                                                                                        
uvlink src = /mnt/build-arm/_deps/uv_link-src                                                                                                                           
-- Found OpenSSL: /usr/lib/x86_64-linux-gnu/libcrypto.so (found version "1.1.1n")

If we can narrow the search to precisely what is generating this Found OpenSSL line that may lead to the root of the problem.

I've pushed the updated Dockerfile.linux-build to the branch so you can repro.

[qrkourier]

I got past the wrong arch libssl.so problem by setting this in the arm toolchain file:

set(OPENSSL_ROOT_DIR /usr/lib/${triple})

Thanks for the hint @sabedevops!

[qrkourier]

Next step is to resolve #543 and I'll do that here in the same branch.