Split meassurements (#2173)

* Split meassurements * Test contracts upload * split ink tests * Add run id as param * Fix path * Use standard runner for codecov job * Test storage consumption * Use standard runners * Split examples into 8 jobs * Fix examples workflow * Runner cleanup * Remove rust info * Fix examples build * Update examples test * Cleanup * Split tests into 8 jobs * Split tests into 6 jobs
use-ink · Apr 8, 2024 · f107a92 · f107a92
1 parent 0bd8391
commit f107a92
Show file tree

Hide file tree

Showing 4 changed files with 112 additions and 68 deletions.
diff --git a/.github/download-artifact/action.yml b/.github/download-artifact/action.yml
@@ -7,6 +7,10 @@ inputs:
   ARTIFACT_NAME:
     description: Name of the artifact to be downloaded
     required: true
+  RUN_ID:
+    description: The Run ID of the workflow from which the artifact is to be downloaded
+    required: false
+    default: ${{ github.event.workflow_run.id }}
   DOWNLOAD_DIR:
     description: Destination directory
     required: false
@@ -26,7 +30,7 @@ runs:
             var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              run_id: ${{ github.event.workflow_run.id }},
+              run_id: ${{ inputs.RUN_ID }},
             });
             const matchArtifact = artifacts.data.artifacts.find((artifact) => artifact.name === artifactName);
 
@@ -43,5 +47,5 @@ runs:
               core.setFailed(`Artifact ${artifactName} not found.`);
             }
 
-      - run: unzip data.zip -d ./${{ inputs.DOWNLOAD_DIR }}
+      - run: unzip -o data.zip -d ./${{ inputs.DOWNLOAD_DIR }}
         shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -321,13 +321,17 @@ jobs:
           done
 
   test:
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest
     needs: [set-image, check]
     defaults:
       run:
         shell: bash
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
+    strategy:
+      fail-fast: false
+      matrix:
+        workspace: [ink, linting]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -339,12 +343,12 @@ jobs:
         with:
           cache-directories: |
             ${{ env.CARGO_TARGET_DIR }}
-            ${{ env.CARGO_TARGET_DIR }}/linting
 
       - name: Rust Info
         uses: ./.github/rust-info
 
       - name: Test
+        if: ${{ matrix.workspace == 'ink' }}
         env:
           # Fix for linking of `linkme` for `cargo test`: https://github.com/dtolnay/linkme/issues/49
           RUSTFLAGS:                   -Clink-arg=-z -Clink-arg=nostart-stop-gc
@@ -356,6 +360,18 @@ jobs:
         run: |
           cargo nextest run --all-features --no-fail-fast --workspace --locked
           cargo test --all-features --no-fail-fast --workspace --doc --locked
+
+      - name: Test Linting
+        if: ${{ matrix.workspace == 'linting' }}
+        env:
+          # Fix for linking of `linkme` for `cargo test`: https://github.com/dtolnay/linkme/issues/49
+          RUSTFLAGS:                   -Clink-arg=-z -Clink-arg=nostart-stop-gc
+          # Since we run the tests with `--all-features` this implies the feature
+          # `ink-fuzz-tests` as well -- i.e. the fuzz tests are run.
+          # There's no way to disable a single feature while enabling all features
+          # at the same time, hence we use this workaround.
+          QUICKCHECK_TESTS:            0
+        run: |
           pushd linting && cargo nextest run --all-features --no-fail-fast --workspace && popd
 
   docs:
@@ -399,7 +415,7 @@ jobs:
           retention-days: 1
 
   codecov:
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest
     needs: [set-image, check, fmt, clippy, clippy-examples, dylint, spellcheck]
     defaults:
       run:
@@ -459,16 +475,48 @@ jobs:
 ### examples
 
   examples-test:
-    runs-on: ubuntu-latest-8-cores
+    runs-on: ubuntu-latest
+    needs: [clippy]
+    strategy:
+      matrix:
+        partition: [1, 2, 3, 4, 5, 6]
+    steps:
+      # We go out of storage on runners
+      - name: Clean runner
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+
+      - name: Cache
+        uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
+        with:
+          cache-directories: ${{ env.CARGO_TARGET_DIR }}
+
+      - name: Test Examples
+        env:
+          # Fix linking of `linkme`: https://github.com/dtolnay/linkme/issues/49
+          RUSTFLAGS:                     -Clink-arg=-z -Clink-arg=nostart-stop-gc
+        uses: docker://paritytech/ci-unified:bullseye-1.75.0
+        with:
+          # run all tests with --all-features, which will run the e2e-tests feature if present
+          args: /bin/bash -c "scripts/for_all_contracts_exec.sh --path integration-tests --ignore public/static-buffer --partition ${{ matrix.partition }}/6 -- cargo test \
+            --all-features --manifest-path {}"
+
+  examples-custom-test:
+    runs-on: ubuntu-latest
     needs: [set-image, clippy]
     defaults:
       run:
         shell: bash
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
-    strategy:
-      matrix:
-        partition: [1, 2, 3, 4]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -483,17 +531,7 @@ jobs:
       - name: Rust Info
         uses: ./.github/rust-info
 
-      - name: Test Examples
-        env:
-          # Fix linking of `linkme`: https://github.com/dtolnay/linkme/issues/49
-          RUSTFLAGS:                     -Clink-arg=-z -Clink-arg=nostart-stop-gc
-        run: |
-          # run all tests with --all-features, which will run the e2e-tests feature if present
-          scripts/for_all_contracts_exec.sh --path integration-tests --ignore public/static-buffer --partition ${{ matrix.partition }}/4 -- cargo test \
-            --all-features --manifest-path {}
-
       - name: Test static-buffer example
-        if: ${{ matrix.partition == 3 }}
         env:
           # Fix linking of `linkme`: https://github.com/dtolnay/linkme/issues/49
           RUSTFLAGS:                     -Clink-arg=-z -Clink-arg=nostart-stop-gc
@@ -503,23 +541,22 @@ jobs:
           INK_STATIC_BUFFER_SIZE=30 cargo test --manifest-path integration-tests/public/static-buffer/Cargo.toml --all-features
 
       - name: Run E2E test with on-chain contract
-        if: ${{ matrix.partition == 4 }}
         env:
-            # Fix linking of `linkme`: https://github.com/dtolnay/linkme/issues/49
-            RUSTFLAGS:                     -Clink-arg=-z -Clink-arg=nostart-stop-gc
+          # Fix linking of `linkme`: https://github.com/dtolnay/linkme/issues/49
+          RUSTFLAGS:                     -Clink-arg=-z -Clink-arg=nostart-stop-gc
         run: |
-            # run flipper E2E test with on-chain contract
-            substrate-contracts-node -lruntime::contracts=debug 2>&1 &
-            cargo contract build --release --manifest-path integration-tests/public/flipper/Cargo.toml
-            export CONTRACT_ADDR_HEX=$(cargo contract instantiate \
-              --manifest-path integration-tests/public/flipper/Cargo.toml \
-              --suri //Alice --args true -x -y --output-json | \
-              jq -r .contract | xargs subkey inspect | grep -o "0x.*" | head -n1)
-            CONTRACTS_NODE_URL=ws://127.0.0.1:9944 cargo test \
-              --features e2e-tests \
-              --manifest-path integration-tests/public/flipper/Cargo.toml \
-              e2e_test_deployed_contract \
-              -- --ignored --nocapture
+          # run flipper E2E test with on-chain contract
+          substrate-contracts-node -lruntime::contracts=debug 2>&1 &
+          cargo contract build --release --manifest-path integration-tests/public/flipper/Cargo.toml
+          export CONTRACT_ADDR_HEX=$(cargo contract instantiate \
+            --manifest-path integration-tests/public/flipper/Cargo.toml \
+            --suri //Alice --args true -x -y --output-json | \
+            jq -r .contract | xargs subkey inspect | grep -o "0x.*" | head -n1)
+          CONTRACTS_NODE_URL=ws://127.0.0.1:9944 cargo test \
+            --features e2e-tests \
+            --manifest-path integration-tests/public/flipper/Cargo.toml \
+            e2e_test_deployed_contract \
+            -- --ignored --nocapture
 
   examples-contract-build:
     runs-on: ubuntu-latest
@@ -622,14 +659,14 @@ jobs:
 # measurements
 
   measurements:
-    needs: [examples-docs, examples-contract-build, examples-test]
+    needs: [examples-docs, examples-contract-build, examples-test, examples-custom-test]
     uses: ./.github/workflows/measurements.yml
 
 # fuzz
 
   fuzz:
     runs-on: ubuntu-latest
-    needs: [set-image, examples-docs, examples-contract-build, examples-test]
+    needs: [set-image, examples-docs, examples-contract-build, examples-test, examples-custom-test]
     if: >
       github.event_name == 'push' &&
         github.ref == 'refs/heads/master'

diff --git a/.github/workflows/measurements.yml b/.github/workflows/measurements.yml
@@ -15,12 +15,24 @@ jobs:
         shell: bash
     container:
       image: paritytech/ci-unified:bullseye-1.75.0
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [master, pull_request]
     steps:
-      - name: Checkout
+      - name: Checkout Pull Request
+        if: ${{ matrix.branch == 'pull_request' }}
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
+      - name: Checkout Master
+        if: ${{ matrix.branch == 'master' }}
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          fetch-depth: 1
+
       - name: Cache
         uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
         with:
@@ -34,41 +46,21 @@ jobs:
           PR_NUMBER:               ${{ github.event.number }}
         run: |
           # Variables
-          MEASUREMENTS_DIR="${GITHUB_WORKSPACE}/../ink-ci-measurements"
+          MEASUREMENTS_DIR="${GITHUB_WORKSPACE}/measurements"
           SCRIPTS_DIR="${GITHUB_WORKSPACE}/scripts"
-          ARTIFACTS_DIR="${GITHUB_WORKSPACE}/artifacts"
-          PR_CONTRACT_SIZES="${MEASUREMENTS_DIR}/pull_request_contract_sizes"
-          UPSTREAM_BRANCH="master"
-          UPSTREAM_CONTRACT_SIZES="${MEASUREMENTS_DIR}/${UPSTREAM_BRANCH}_contract_sizes"
-          UPSTREAM_REPO="https://github.com/paritytech/ink.git"
-          UPSTREAM_DIR="${MEASUREMENTS_DIR}/ink-${UPSTREAM_BRANCH}"
-          CONTRACT_SIZES_DIFF="${MEASUREMENTS_DIR}/contract_sizes_diff.md"
+          CONTRACT_SIZES="${MEASUREMENTS_DIR}/${{ matrix.branch }}_contract_sizes"
 
           # Measure contract sizes for the current branch
           mkdir ${MEASUREMENTS_DIR}
-          ${SCRIPTS_DIR}/for_all_contracts_exec.sh --path integration-tests --quiet -- ${SCRIPTS_DIR}/contract_size.sh {} > ${PR_CONTRACT_SIZES}
-          sed -ie 's/^integration-tests\/\(public\/\|internal\/\)\?//' ${PR_CONTRACT_SIZES}
-          # Measure contract sizes for the upstream branch
-          git clone --depth 1 --branch ${UPSTREAM_BRANCH} ${UPSTREAM_REPO} ${UPSTREAM_DIR}
-          pushd ${UPSTREAM_DIR}
-          ${SCRIPTS_DIR}/for_all_contracts_exec.sh --path integration-tests --quiet -- ${SCRIPTS_DIR}/contract_size.sh {} > ${UPSTREAM_CONTRACT_SIZES}
-          sed -ie 's/^integration-tests\/\(public\/\|internal\/\)\?//' ${UPSTREAM_CONTRACT_SIZES}
-          popd
-
-          # Build the comparison table
-          ${SCRIPTS_DIR}/contract_sizes_diff.sh ${UPSTREAM_CONTRACT_SIZES} ${PR_CONTRACT_SIZES} > ${CONTRACT_SIZES_DIFF}
-          cat ${CONTRACT_SIZES_DIFF}
+          ${SCRIPTS_DIR}/for_all_contracts_exec.sh --path integration-tests --quiet -- ${SCRIPTS_DIR}/contract_size.sh {} > ${CONTRACT_SIZES}
+          sed -ie 's/^integration-tests\/\(public\/\|internal\/\)\?//' ${CONTRACT_SIZES}
 
-          # Create job artifact
-          mkdir ${ARTIFACTS_DIR}
-          mv ${CONTRACT_SIZES_DIFF} ${ARTIFACTS_DIR}/
           CARGO_CONTRACT_VERSION=$(cargo-contract --version | egrep --only-matching "cargo-contract.* .*-x86" | sed -s 's/-x86//')
-          echo "CARGO_CONTRACT_VERSION=\"${CARGO_CONTRACT_VERSION}\"" > ${ARTIFACTS_DIR}/context.out
-          echo "PR_NUMBER=\"${PR_NUMBER}\"" >> ${ARTIFACTS_DIR}/context.out
-          rm -rf ${MEASUREMENTS_DIR}
+          echo "CARGO_CONTRACT_VERSION=\"${CARGO_CONTRACT_VERSION}\"" > ${MEASUREMENTS_DIR}/context.out
+          echo "PR_NUMBER=\"${PR_NUMBER}\"" >> ${MEASUREMENTS_DIR}/context.out
 
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ github.job }}-data
-          path: ./artifacts
+          name: ${{ github.job }}-${{ matrix.branch }}-data
+          path: ./measurements
           retention-days: 1
diff --git a/.github/workflows/submit-contract-sizes.yml b/.github/workflows/submit-contract-sizes.yml
@@ -25,11 +25,23 @@ jobs:
         with:
           fetch-depth: 100
 
-      - name: Download Contract Sizes
+      - name: Download Master Contract Sizes
         uses: ./.github/download-artifact
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          ARTIFACT_NAME: contract-sizes-data
+          ARTIFACT_NAME: contract-sizes-master-data
+
+      - name: Download Pull-Request Contract Sizes
+        uses: ./.github/download-artifact
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ARTIFACT_NAME: contract-sizes-pull_request-data
+
+      - name: Collect Contract Sizes
+        run: |
+          # Build the comparison table
+          ./scripts/contract_sizes_diff.sh master_contract_sizes pull_request_contract_sizes > contract_sizes_diff.md
+          cat contract_sizes_diff.md
 
       - name: Submit Comment
         env:
@@ -45,4 +57,3 @@ jobs:
           # Submit the comparison table as a comment to the PR
           echo "Submitting contract sizes diff to ${PR_COMMENTS_URL}"
           GITHUB_PR_TOKEN=${GITHUB_PR_TOKEN} .github/scripts/contract_sizes_submit.sh ${PR_COMMENTS_URL} ${WORKFLOW_URL} ${CARGO_CONTRACT_VERSION} < ./contract_sizes_diff.md
-