Skip to content

Commit

Permalink
Fully split native image and mvn build from the cpp and python bindin…
Browse files Browse the repository at this point in the history
…gs part.

Signed-off-by: Bertrand Rix <[email protected]>
  • Loading branch information
obrix committed Aug 2, 2024
1 parent 69b2ad6 commit 8d59b94
Show file tree
Hide file tree
Showing 7 changed files with 301 additions and 80 deletions.
135 changes: 135 additions & 0 deletions .github/workflows/release-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
name: Release

on:
release:
types:
- created

jobs:
manylinux_build:
name: Build linux ${{ matrix.python.name }} wheel
runs-on: ubuntu-latest
container: quay.io/pypa/manylinux2014_x86_64:2024-07-01-8dac23b
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
strategy:
matrix:
python:
- {
name: cp38,
abi: cp38,
version: '3.8',
}

steps:
- name: Install Maven
run: |
curl --fail --silent --show-error https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz -o /tmp/apache-maven-3.9.6-bin.tar.gz
tar xf /tmp/apache-maven-*.tar.gz -C /opt
echo /opt/apache-maven-3.9.6/bin >> $GITHUB_PATH
- name: Setup GraalVM
uses: graalvm/[email protected] # !!! this is last version compatible with manylinux 2014
with:
java-version: '17'
distribution: 'graalvm'
github-token: ${{ secrets.GITHUB_TOKEN }}

- name: Checkout sources
uses: actions/checkout@v3
with:
submodules: true

- name: Setup path
run: echo "/opt/python/${{ matrix.python.name }}-${{ matrix.python.abi }}/bin/" >> $GITHUB_PATH

- name: Install dependencies
run: pip3 install -r requirements.txt

- name: Build wheel
run: |
python3 setup.py bdist_wheel
auditwheel repair dist/*.whl
- name: Install wheel
run: pip3 install wheelhouse/*.whl --user

- name: Run basic pypowsybl import
working-directory: ./tests
run: python3 basic_import_test.py

- name: Upload binaries
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ github.event.release.upload_url }}
asset_path: ./dist/binaries.zip
asset_name: binaries-${{ github.event.release.tag_name }}-linux.zip
asset_content_type: application/zip

macos_windows_build:
name: Build ${{ matrix.config.name }} ${{ matrix.python.name }} wheel
runs-on: ${{ matrix.config.os }}
strategy:
matrix:
config:
- {
name: darwin,
os: macos-12,
}
- {
name: windows,
os: windows-2022,
}
python:
- {
name: cp38,
version: '3.8',
}

steps:
- name: Setup GraalVM
uses: graalvm/setup-graalvm@v1
with:
java-version: '17'
distribution: 'graalvm'
github-token: ${{ secrets.GITHUB_TOKEN }}

- name: Checkout sources
uses: actions/checkout@v3
with:
submodules: true

- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python.version }}

- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
- name: Build wheel
env:
MACOSX_DEPLOYMENT_TARGET: "10.16" # to ensure pip finds wheel when Big Sur is configured to return 10.16 as version instead of 11.0
run: python setup.py bdist_wheel

- name: Install wheel
shell: bash
run: python -m pip install dist/*.whl --user

- name: Run basic pypowsybl import
working-directory: ./tests
run: python3 basic_import_test.py

- name: Upload binaries
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ github.event.release.upload_url }}
asset_path: ./dist/binaries.zip
asset_name: binaries-${{ github.event.release.tag_name }}-${{ matrix.config.name }}.zip
asset_content_type: application/zip
27 changes: 22 additions & 5 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,32 @@
cmake_minimum_required(VERSION 3.14)
project(pypowsybl)

set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)

option(BUILD_PYPOWSYBL_JAVA "Build graalvm based powsybl java c api" ON)
option(BUILD_POWSYBL_CPP "Build cpp powsybl api" ON)
option(BUILD_PYTHON_BINDINGS "Build python bindings" ON)

# change shared library rpath to resolve java library in same directory
# only works on linux
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_INSTALL_RPATH "$ORIGIN")

#native-image math-native and powsybl-cpp targets
add_subdirectory(powsybl-cpp)
add_subdirectory(lib/pybind11)
add_subdirectory(pypowsybl-cpp)
if(BUILD_PYPOWSYBL_JAVA)
#Build mvn, native-image and math-native targets
add_subdirectory(pypowsybl-java)
set(PYPOWSYBL_JAVA_LIBRARY_DIR ${PYPOWSYBL_JAVA_BIN_DIR})
set(PYPOWSYBL_JAVA_INCLUDE_DIR ${PYPOWSYBL_JAVA_BIN_DIR})
endif()

if(BUILD_POWSYBL_CPP)
#Build powsybl cpp target
add_subdirectory(powsybl-cpp)
endif()

if(BUILD_PYTHON_BINDINGS)
#Build pythin bindings
add_subdirectory(lib/pybind11)
add_subdirectory(pypowsybl-cpp)
endif()
89 changes: 17 additions & 72 deletions cpp/powsybl-cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,105 +7,50 @@
cmake_minimum_required(VERSION 3.14)
project(powsybl-cpp)

set(CMAKE_CXX_STANDARD 17)
include(ExternalProject)

if(NOT PYPOWSYBL_JAVA_LIBRARY_DIR OR NOT PYPOWSYBL_JAVA_INCLUDE_DIR)
message( FATAL_ERROR "PYPOWSYBL_JAVA_LIBRARY_DIR or PYPOWSYBL_JAVA_INCLUDE_DIR is undefined. Cannot build powsybl-cpp")
endif()

# Enable static linkage to prevent any future runtime binary compatibility issue
if(MSVC)
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif()

include(ExternalProject)

set(POWSYBL_CPP_LIB ${CMAKE_SHARED_LIBRARY_PREFIX}powsybl-cpp${CMAKE_SHARED_LIBRARY_SUFFIX})

if(NOT DEFINED PYPOWSYBL_JAVA_SRC_DIR)
set(PYPOWSYBL_JAVA_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../java)
endif()
set(PYPOWSYBL_JAVA_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}/../java)
set(PYPOWSYBL_JAVA_OLD_LIB pypowsybl-java${CMAKE_SHARED_LIBRARY_SUFFIX})
set(PYPOWSYBL_JAVA_LIB ${CMAKE_SHARED_LIBRARY_PREFIX}pypowsybl-java${CMAKE_SHARED_LIBRARY_SUFFIX})

if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64" OR ${CMAKE_SYSTEM_PROCESSOR} STREQUAL "AMD64")
set(OS_BITS 64)
elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "arm64")
set(OS_BITS arm64)
else()
set(OS_BITS 32)
endif()

if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(POWSYBL_MATH_NATIVE_LIB math.dll)
set(POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR natives/windows_${OS_BITS})
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
set(POWSYBL_MATH_NATIVE_LIB libmath.so)
set(POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR natives/linux_${OS_BITS})
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
set(POWSYBL_MATH_NATIVE_LIB libmath.dylib)
set(POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR natives/osx_${OS_BITS})
else()
message(FATAL_ERROR "System not supported: ${CMAKE_SYSTEM_NAME}")
endif()

# on MacOS, java library is created with an absolute path id, we need to fix it using install_name_tool before
# linking with our shared library
if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
set(NATIVE_IMAGE_INSTALL_EXTRA_COMMAND COMMAND install_name_tool -id @loader_path/${PYPOWSYBL_JAVA_LIB} ${PYPOWSYBL_JAVA_BIN_DIR}/${PYPOWSYBL_JAVA_LIB})
set(POWSYBL_CPP_INSTALL_EXTRA_COMMAND COMMAND install_name_tool -id @loader_path/${POWSYBL_CPP_LIB} $<TARGET_FILE:powsybl-cpp>)
set(MATH_NATIVE_INSTALL_EXTRA_COMMAND COMMAND install_name_tool -id @loader_path/${POWSYBL_MATH_NATIVE_LIB} ${CMAKE_CURRENT_BINARY_DIR}/${POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR}/${POWSYBL_MATH_NATIVE_LIB})
endif()

ExternalProject_Add(mvn
SOURCE_DIR ${PYPOWSYBL_JAVA_SRC_DIR}
PATCH_COMMAND mvn --batch-mode clean package
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
)

if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
set(NATIVE_IMAGE_BUILD_OPTIONS "-H:GenerateDebugInfo=1" "-Ob")
endif()

# as of GraalVM 23.0.0 G1 GC is only supported on Linux ADM64
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux" AND ${OS_BITS} STREQUAL "64")
set(NATIVE_IMAGE_GC "G1")
else()
set(NATIVE_IMAGE_GC "serial")
endif()

# as GraalVm does not follow same library naming convention (lib prefix is missing on Linux and MacOS) we need to rename
# it in the install command step
ExternalProject_Add(native-image
DEPENDS mvn
SOURCE_DIR ${PYPOWSYBL_JAVA_BIN_DIR}
DOWNLOAD_COMMAND ""
PATCH_COMMAND $ENV{JAVA_HOME}/bin/native-image ${NATIVE_IMAGE_BUILD_OPTIONS} --class-path ${PYPOWSYBL_JAVA_SRC_DIR}/target/pypowsybl-java.jar${EXTRA_JARS} -march=compatibility --no-fallback --shared --gc=${NATIVE_IMAGE_GC} -H:Name=pypowsybl-java -H:CLibraryPath=${CMAKE_CURRENT_SOURCE_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ${PYPOWSYBL_JAVA_BIN_DIR}/${PYPOWSYBL_JAVA_OLD_LIB} ${PYPOWSYBL_JAVA_BIN_DIR}/${PYPOWSYBL_JAVA_LIB} ${NATIVE_IMAGE_INSTALL_EXTRA_COMMAND}
)

# extract powsybl math native from jar for current platform
# powsybl-math-native.jar has been previously copied by maven build
add_custom_target(math-native ALL COMMAND ${CMAKE_COMMAND} -E tar x ${PYPOWSYBL_JAVA_SRC_DIR}/target/dependency/powsybl-math-native.jar ${POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR}/${POWSYBL_MATH_NATIVE_LIB} ${MATH_NATIVE_INSTALL_EXTRA_COMMAND})

include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${PYPOWSYBL_JAVA_BIN_DIR})

link_directories(${PYPOWSYBL_JAVA_BIN_DIR})
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${PYPOWSYBL_JAVA_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../pypowsybl-java)
link_directories(${PYPOWSYBL_JAVA_LIBRARY_DIR})

set(POWSYBL_CPP_SOURCES "powsybl-cpp.cpp")

add_library(powsybl-cpp SHARED ${POWSYBL_CPP_SOURCES})
add_dependencies(powsybl-cpp native-image)

if (TARGET native-image)
add_dependencies(powsybl-cpp native-image)
endif()

if(DEFINED CMAKE_LIBRARY_OUTPUT_DIRECTORY)
add_custom_command(TARGET powsybl-cpp POST_BUILD ${POWSYBL_CPP_INSTALL_EXTRA_COMMAND} COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:powsybl-cpp> ${PYPOWSYBL_JAVA_BIN_DIR}/${PYPOWSYBL_JAVA_LIB} ${CMAKE_CURRENT_BINARY_DIR}/${POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR}/${POWSYBL_MATH_NATIVE_LIB} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
add_custom_command(TARGET powsybl-cpp POST_BUILD ${POWSYBL_CPP_INSTALL_EXTRA_COMMAND} COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:powsybl-cpp> $<TARGET_LINKER_FILE:powsybl-cpp> ${PYPOWSYBL_JAVA_BIN_DIR}/${PYPOWSYBL_JAVA_LIB} ${CMAKE_CURRENT_BINARY_DIR}/${POWSYBL_MATH_NATIVE_JAR_ENTRY_DIR}/${POWSYBL_MATH_NATIVE_LIB} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
endif(DEFINED CMAKE_LIBRARY_OUTPUT_DIRECTORY)

# Fix related to issue describred here https://github.com/actions/runner-images/issues/10004#issuecomment-2156109231
# Fix related to issue described here https://github.com/actions/runner-images/issues/10004#issuecomment-2156109231
# Should fix incompatibility between MSVC runtime 14.40.XXX and previous version
if(MSVC)
target_compile_definitions(powsybl-cpp
PRIVATE _DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR)
endif()

target_link_libraries(powsybl-cpp PUBLIC ${PYPOWSYBL_JAVA_LIB})
install(TARGETS powsybl-cpp)
6 changes: 3 additions & 3 deletions cpp/pypowsybl-cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ if(MSVC)
endif()

set(POWSYBL_CPP_SOURCE_DIR "../powsybl-cpp")
set(PYPOWSYBL_JAVA_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}/../java)
include_directories(${POWSYBL_CPP_SOURCE_DIR} ${PYPOWSYBL_JAVA_BIN_DIR})
set(PYPOWSYBL_JAVA_SOURCE_DIR "../pypowsybl-java")
include_directories(${POWSYBL_CPP_SOURCE_DIR} ${PYPOWSYBL_JAVA_SOURCE_DIR} ${PYPOWSYBL_JAVA_INCLUDE_DIR})

link_directories(${PYPOWSYBL_JAVA_BIN_DIR})
link_directories(${PYPOWSYBL_JAVA_LIBRARY_DIR})
pybind11_add_module(_pypowsybl pylogging.cpp bindings.cpp)

add_dependencies(_pypowsybl native-image math-native)
Expand Down
Loading

0 comments on commit 8d59b94

Please sign in to comment.