Merge pull request #5 from swiftwasm/katei/5.9-scheme

Add 5.9 scheme
swiftwasm · Jun 30, 2023 · 84fb7f6 · 84fb7f6
2 parents 58d9161 + cfedec5
commit 84fb7f6
Show file tree

Hide file tree

Showing 63 changed files with 4,147 additions and 20 deletions.
diff --git a/.github/scripts/build-matrix.rb b/.github/scripts/build-matrix.rb
@@ -99,11 +99,18 @@ def main
     end
   end
 
-  schemes = ["main"]
+  schemes = ["main", "release-5.9"]
 
   matrix_entries = schemes.flat_map do |scheme|
+    if scheme == "main"
+      toolchain_channel = "DEVELOPMENT"
+    elsif scheme.start_with?("release-")
+      toolchain_channel = scheme.sub("release-", "")
+    else
+      raise "Unknown scheme: #{scheme}"
+    end
     matrix_entries.map do |entry|
-      entry.merge("scheme": scheme)
+      entry.merge("scheme": scheme, toolchain_channel: toolchain_channel)
     end
   end
 

diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml
@@ -81,7 +81,6 @@ jobs:
 
   build-toolchain:
     env:
-      TOOLCHAIN_CHANNEL: DEVELOPMENT
       DEVELOPER_DIR: /Applications/Xcode_14.2.app/Contents/Developer/
       # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#cache-segment-restore-timeout
       SEGMENT_DOWNLOAD_TIMEOUT_MINS: 30
@@ -92,7 +91,7 @@ jobs:
       matrix:
         include: ${{ fromJSON(needs.build-matrix.outputs.entries) }}
 
-    name: Target ${{ matrix.target }}
+    name: Target ${{ matrix.scheme }}/${{ matrix.target }}
     timeout-minutes: 0
     runs-on: ${{ matrix.agent_query }}
     steps:
@@ -129,7 +128,7 @@ jobs:
       - name: Prepare sccache timestamp
         id: cache_key
         run: |
-          export SCCACHE_KEY_PREFIX="${{ matrix.target }}-${{ env.TOOLCHAIN_CHANNEL }}-sccache-v14-"
+          export SCCACHE_KEY_PREFIX="${{ matrix.target }}-${{ matrix.toolchain_channel }}-sccache-v14-"
           echo "SCCACHE_KEY_PREFIX=$SCCACHE_KEY_PREFIX" >> $GITHUB_OUTPUT
           echo "SCCACHE_KEY=$SCCACHE_KEY_PREFIX$(date +'%Y-%m-%d-%I-%M-%S')" >> $GITHUB_OUTPUT
 
@@ -156,7 +155,7 @@ jobs:
         if: ${{ steps.build-cache.outputs.cache-hit }}
 
       - run: ./swiftwasm-build/tools/build/install-build-sdk.sh ${{ matrix.scheme }}
-      - run: ./swiftwasm-build/tools/git-swift-workspace --skip-history --scheme ${{ matrix.scheme }}
+      - run: ./swiftwasm-build/tools/git-swift-workspace --skip-history --verbose --scheme ${{ matrix.scheme }}
 
       - name: Install Homebrew dependencies
         if: ${{ startsWith(matrix.build_os, 'macos-') }}
@@ -172,13 +171,12 @@ jobs:
             --cap-add=SYS_PTRACE \
             --security-opt seccomp=unconfined \
             -dit \
-            -e TOOLCHAIN_CHANNEL=${{ env.TOOLCHAIN_CHANNEL }} \
             -w /home/build-user/ \
             -v ${{ github.workspace }}:/source \
             -v oss-swift-package:/home/build-user \
             ${{ matrix.container }}
           docker exec swiftwasm-ci-buildbot /bin/bash -lc 'env; cp -r /source/* /home/build-user/; ./swiftwasm-build/tools/build/ci.sh ${{ matrix.scheme }}'
-          docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ env.TOOLCHAIN_CHANNEL }}-SNAPSHOT-${{ matrix.target }}.tar.gz .
+          docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz .
 
       - name: Extract build-cache from Docker container
         if: ${{ matrix.container != null && always() }}
@@ -203,7 +201,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.target }}-installable
-          path: swift-wasm-${{ env.TOOLCHAIN_CHANNEL }}-SNAPSHOT-${{ matrix.target }}.tar.gz
+          path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz
 
       - name: Run stdlib tests for wasi-wasm32
         if: ${{ matrix.run_stdlib_test && matrix.container != null }}
@@ -237,8 +235,8 @@ jobs:
       - name: Prepare E2E test
         run: |
           INSTALL_DIR=$(mktemp -d)
-          tar xf swift-wasm-$TOOLCHAIN_CHANNEL-SNAPSHOT-${{ matrix.target }}.tar.gz -C "$INSTALL_DIR"
-          echo "TOOLCHAIN=$(find "$INSTALL_DIR" -name "swift-wasm-$TOOLCHAIN_CHANNEL-*" -type d | head -n1)" >> $GITHUB_ENV
+          tar xf swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-${{ matrix.target }}.tar.gz -C "$INSTALL_DIR"
+          echo "TOOLCHAIN=$(find "$INSTALL_DIR" -name "swift-wasm-${{ matrix.toolchain_channel }}-*" -type d | head -n1)" >> $GITHUB_ENV
       - name: Build hello.wasm
         shell: bash
         if: ${{ matrix.build_hello_wasm }}

diff --git a/schemes/release-5.9/manifest.json b/schemes/release-5.9/manifest.json
@@ -0,0 +1,9 @@
+{
+  "update-checkout-scheme": "release/5.9",
+  "base-tag": "swift-5.9-DEVELOPMENT-SNAPSHOT-2023-06-17-a",
+  "repos": {
+    "swift-corelibs-foundation": "660624d0220c0efbef054f4919f9f01fd3866be1",
+    "swift-corelibs-xctest": "77bc9f5386ee8a2a4e8da5ac30e846b451d101b6"
+  },
+  "swift-org-download-channel": "swift-5.9-branch"
+}
diff --git a/schemes/release-5.9/swift/0001-wasm-Add-metadata-registration-for-WebAssembly.patch b/schemes/release-5.9/swift/0001-wasm-Add-metadata-registration-for-WebAssembly.patch
@@ -0,0 +1,128 @@
+From 7ba73fc0c8360ff659eaf1bb140cf10303a0c779 Mon Sep 17 00:00:00 2001
+From: Yuta Saito <[email protected]>
+Date: Sun, 11 Jun 2023 11:44:24 +0000
+Subject: [PATCH] [wasm] Add metadata registration for WebAssembly
+
+This patch adds the metadata registration for the wasm targets, and also
+adds build support for it.
+---
+ stdlib/public/runtime/CMakeLists.txt          | 26 ++++++++++++++++---
+ .../{SwiftRT-ELF.cpp => SwiftRT-ELF-WASM.cpp} | 15 +++++++++--
+ 2 files changed, 35 insertions(+), 6 deletions(-)
+ rename stdlib/public/runtime/{SwiftRT-ELF.cpp => SwiftRT-ELF-WASM.cpp} (82%)
+
+diff --git a/stdlib/public/runtime/CMakeLists.txt b/stdlib/public/runtime/CMakeLists.txt
+index 3771d972d54..38de01cd2a8 100644
+--- a/stdlib/public/runtime/CMakeLists.txt
++++ b/stdlib/public/runtime/CMakeLists.txt
+@@ -88,7 +88,7 @@ set(swift_runtime_backtracing_sources
+ # Acknowledge that the following sources are known.
+ set(LLVM_OPTIONAL_SOURCES
+     SwiftRT-COFF.cpp
+-    SwiftRT-ELF.cpp
++    SwiftRT-ELF-WASM.cpp
+     ${swift_runtime_sources}
+     ${swift_runtime_objc_sources}
+     ${swift_runtime_leaks_sources}
+@@ -139,11 +139,14 @@ add_swift_target_library(swiftRuntime OBJECT_LIBRARY
+
+ set(ELFISH_SDKS)
+ set(COFF_SDKS)
++set(WASM_SDKS)
+ foreach(sdk ${SWIFT_SDKS})
+   if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF")
+     list(APPEND ELFISH_SDKS ${sdk})
+   elseif("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF")
+     list(APPEND COFF_SDKS ${sdk})
++  elseif("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
++    list(APPEND WASM_SDKS ${sdk})
+   endif()
+ endforeach()
+
+@@ -152,7 +155,7 @@ endforeach()
+ # with LTO, force swift runtime to compile without LTO for Linux.
+ add_swift_target_library(swiftImageRegistrationObjectELF
+                   OBJECT_LIBRARY IS_STDLIB IS_STDLIB_CORE
+-                  SwiftRT-ELF.cpp
++                  SwiftRT-ELF-WASM.cpp
+                   C_COMPILE_FLAGS ${SWIFT_RUNTIME_CORE_CXX_FLAGS}
+                   C_COMPILE_FLAGS_LINUX -fno-lto
+                   LINK_FLAGS ${SWIFT_RUNTIME_CORE_LINK_FLAGS}
+@@ -170,17 +173,32 @@ add_swift_target_library(swiftImageRegistrationObjectCOFF
+                   SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+                   INSTALL_IN_COMPONENT none)
+
++add_swift_target_library(swiftImageRegistrationObjectWASM
++                  OBJECT_LIBRARY IS_STDLIB IS_STDLIB_CORE
++                  SwiftRT-ELF-WASM.cpp
++                  C_COMPILE_FLAGS ${SWIFT_RUNTIME_CORE_CXX_FLAGS}
++                  LINK_FLAGS ${SWIFT_RUNTIME_CORE_LINK_FLAGS}
++                  TARGET_SDKS ${WASM_SDKS}
++                  SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
++                  INSTALL_IN_COMPONENT none)
++
+ foreach(sdk ${SWIFT_SDKS})
+   foreach(arch ${SWIFT_SDK_${sdk}_ARCHITECTURES})
+     set(arch_subdir "${SWIFT_SDK_${sdk}_LIB_SUBDIR}/${arch}")
+     set(arch_suffix "${SWIFT_SDK_${sdk}_LIB_SUBDIR}-${arch}")
+
+     if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF" OR
+-       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF")
++       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF" OR
++       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
+       # TODO(compnerd) switch to the generator expression when cmake is upgraded
+       # to a version which supports it.
+       # set(swiftrtObject "$<TARGET_OBJECTS:swiftImageRegistrationObject${SWIFT_SDK_${sdk}_OBJECT_FORMAT}-${arch_suffix}>")
+-      set(swiftrtObject ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/swiftImageRegistrationObject${SWIFT_SDK_${sdk}_OBJECT_FORMAT}-${arch_suffix}.dir/SwiftRT-${SWIFT_SDK_${sdk}_OBJECT_FORMAT}.cpp${CMAKE_C_OUTPUT_EXTENSION})
++      set(swiftrtSourceName SwiftRT-${SWIFT_SDK_${sdk}_OBJECT_FORMAT}.cpp)
++      if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF" OR
++         "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
++       set(swiftrtSourceName SwiftRT-ELF-WASM.cpp)
++      endif()
++      set(swiftrtObject ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/swiftImageRegistrationObject${SWIFT_SDK_${sdk}_OBJECT_FORMAT}-${arch_suffix}.dir/${swiftrtSourceName}${CMAKE_C_OUTPUT_EXTENSION})
+
+       if(sdk STREQUAL WINDOWS)
+         set(extension .obj)
+diff --git a/stdlib/public/runtime/SwiftRT-ELF.cpp b/stdlib/public/runtime/SwiftRT-ELF-WASM.cpp
+similarity index 82%
+rename from stdlib/public/runtime/SwiftRT-ELF.cpp
+rename to stdlib/public/runtime/SwiftRT-ELF-WASM.cpp
+index 1b9bf3b9aa7..e68103a6a67 100644
+--- a/stdlib/public/runtime/SwiftRT-ELF.cpp
++++ b/stdlib/public/runtime/SwiftRT-ELF-WASM.cpp
+@@ -1,4 +1,4 @@
+-//===--- SwiftRT-ELF.cpp --------------------------------------------------===//
++//===--- SwiftRT-ELF-WASM.cpp ---------------------------------------------===//
+ //
+ // This source file is part of the Swift.org open source project
+ //
+@@ -16,14 +16,25 @@
+ #include <cstddef>
+ #include <new>
+
++#if defined(__ELF__)
+ extern "C" const char __dso_handle[];
++#elif defined(__wasm__)
++// NOTE: Multi images in a single process is not yet
++// stabilized in WebAssembly toolchain outside of Emscripten.
++static constexpr const void *__dso_handle = nullptr;
++#endif
+
+ // Create empty sections to ensure that the start/stop symbols are synthesized
+ // by the linker.  Otherwise, we may end up with undefined symbol references as
+ // the linker table section was never constructed.
++#if defined(__ELF__)
++# define DECLARE_EMPTY_METADATA_SECTION(name) __asm__("\t.section " #name ",\"a\"\n");
++#elif defined(__wasm__)
++# define DECLARE_EMPTY_METADATA_SECTION(name) __asm__("\t.section " #name ",\"\",@\n");
++#endif
+
+ #define DECLARE_SWIFT_SECTION(name)                                                          \
+-  __asm__("\t.section " #name ",\"a\"\n");                                                   \
++  DECLARE_EMPTY_METADATA_SECTION(name)                                                                \
+   __attribute__((__visibility__("hidden"),__aligned__(1))) extern const char __start_##name; \
+   __attribute__((__visibility__("hidden"),__aligned__(1))) extern const char __stop_##name;
+
+-- 
+2.40.0
+
diff --git a/schemes/release-5.9/swift/0002-wasm-cmake-HACK-Skip-building-swift-reflection-test-.patch b/schemes/release-5.9/swift/0002-wasm-cmake-HACK-Skip-building-swift-reflection-test-.patch
@@ -0,0 +1,44 @@
+From 8654cbe30eb06febfc29cbb0256af06942d10fe0 Mon Sep 17 00:00:00 2001
+From: Yuta Saito <[email protected]>
+Date: Tue, 20 Jun 2023 06:49:07 +0000
+Subject: [PATCH] [wasm][cmake] HACK: Skip building swift-reflection-test for
+ now
+
+The reason why this change is needed is unclear to me...
+---
+ CMakeLists.txt                | 5 +++++
+ stdlib/private/CMakeLists.txt | 2 +-
+ 2 files changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 563d40ee885..309a32524b6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -560,6 +560,11 @@ option(SWIFT_STDLIB_ENABLE_OBJC_INTEROP
+        "Should stdlib be built with Obj-C interop."
+        "${SWIFT_STDLIB_ENABLE_OBJC_INTEROP_default}")
+
++# FIXME(wasm) Reflection tests are temporalily disabled due to lack of linker features
++option(SWIFTWASM_DISABLE_REFLECTION_TEST
++    "Disable building swift-reflection-test for WebAssembly build"
++    FALSE)
++
+ #
+ # User-configurable experimental options.  Do not use in production builds.
+ #
+diff --git a/stdlib/private/CMakeLists.txt b/stdlib/private/CMakeLists.txt
+index 05ef455b755..704d96f9248 100644
+--- a/stdlib/private/CMakeLists.txt
++++ b/stdlib/private/CMakeLists.txt
+@@ -28,7 +28,7 @@ if(SWIFT_BUILD_SDK_OVERLAY)
+   endif()
+   # Currently SwiftReflectionTest cannot be built on Windows, due to
+   # dependencies on POSIX symbols
+-  if (SWIFT_INCLUDE_TESTS AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Windows"))
++  if (SWIFT_INCLUDE_TESTS AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Windows") AND NOT SWIFTWASM_DISABLE_REFLECTION_TEST)
+     add_subdirectory(SwiftReflectionTest)
+   endif()
+ endif()
+-- 
+2.40.0
+
diff --git a/schemes/release-5.9/swift/0003-wasm-cmake-HACK-Tweak-standalone-stdlib-build-config.patch b/schemes/release-5.9/swift/0003-wasm-cmake-HACK-Tweak-standalone-stdlib-build-config.patch
@@ -0,0 +1,44 @@
+From ffb195ae40e8fed2b58bca2932c29a3384195ba4 Mon Sep 17 00:00:00 2001
+From: Yuta Saito <[email protected]>
+Date: Tue, 20 Jun 2023 06:58:41 +0000
+Subject: [PATCH] [wasm][cmake] HACK: Tweak standalone stdlib build
+ configuration
+
+This commit disables remote mirrors due to cross-compilation issues, and
+turns off several additional features that are not yet supported on WASI.
+---
+ cmake/caches/Runtime-WASI-wasm32.cmake | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/cmake/caches/Runtime-WASI-wasm32.cmake b/cmake/caches/Runtime-WASI-wasm32.cmake
+index d96dc885022..2938f4a8f60 100644
+--- a/cmake/caches/Runtime-WASI-wasm32.cmake
++++ b/cmake/caches/Runtime-WASI-wasm32.cmake
+@@ -8,6 +8,7 @@ set(SWIFT_SDKS WASI CACHE STRING "")
+ set(SWIFT_INCLUDE_TOOLS NO CACHE BOOL "")
+ set(SWIFT_INCLUDE_TESTS TRUE CACHE BOOL "")
+ set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "")
++set(SWIFT_BUILD_REMOTE_MIRROR NO CACHE BOOL "")
+
+ set(SWIFT_BUILD_SOURCEKIT NO CACHE BOOL "")
+ set(SWIFT_ENABLE_SOURCEKIT_TESTS NO CACHE BOOL "")
+@@ -18,8 +19,16 @@ set(SWIFTWASM_DISABLE_REFLECTION_TEST YES CACHE BOOL "")
+ set(SWIFT_BUILD_STATIC_STDLIB YES CACHE BOOL "")
+ set(SWIFT_BUILD_DYNAMIC_STDLIB NO CACHE BOOL "")
+ set(SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY YES CACHE BOOL "")
++# TODO(katei): This should get turned off, as this is not an ABI stable platform.
++# But current CMake build system doesn't support SWIFT_STDLIB_STABLE_ABI=NO
++set(SWIFT_STDLIB_STABLE_ABI YES CACHE BOOL "")
+
+ # build with the host compiler
+ set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "")
+
+ set(SWIFT_STDLIB_SINGLE_THREADED_CONCURRENCY TRUE CACHE BOOL "")
++set(SWIFT_THREADING_PACKAGE TRUE CACHE STRING "none")
++
++set(SWIFT_STDLIB_SUPPORTS_BACKTRACE_REPORTING FALSE CACHE BOOL "")
++set(SWIFT_STDLIB_HAS_DLADDR FALSE CACHE BOOL "")
++set(SWIFT_STDLIB_COMPACT_ABSOLUTE_FUNCTION_POINTER TRUE CACHE BOOL "")
+-- 
+2.40.0
+
diff --git a/schemes/release-5.9/swift/0004-wasm-cmake-HACK-link-wasi-emulated-mman-to-host-exec.patch b/schemes/release-5.9/swift/0004-wasm-cmake-HACK-link-wasi-emulated-mman-to-host-exec.patch
@@ -0,0 +1,27 @@
+From 68ee54c737c46393f6c070af057b4a9d3f4d49f8 Mon Sep 17 00:00:00 2001
+From: Yuta Saito <[email protected]>
+Date: Tue, 20 Jun 2023 07:04:03 +0000
+Subject: [PATCH] [wasm][cmake] HACK: link wasi-emulated-mman to host
+ executable
+
+The reason why this change is needed is unclear to me...
+---
+ cmake/modules/AddSwift.cmake | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake
+index 248178cd470..cb8b7078b78 100644
+--- a/cmake/modules/AddSwift.cmake
++++ b/cmake/modules/AddSwift.cmake
+@@ -397,6 +397,8 @@ function(_add_host_variant_link_flags target)
+       cxx_link_libraries)
+     target_link_libraries(${target} PRIVATE
+       ${cxx_link_libraries})
++  elseif("${LFLAGS_SDK}" STREQUAL "WASI")
++    list(APPEND result "-Wl,wasi-emulated-mman")
+   else()
+     # If lto is enabled, we need to add the object path flag so that the LTO code
+     # generator leaves the intermediate object file in a place where it will not
+-- 
+2.40.0
+