Develop

.github/workflows/ci-slow.yml

@@ -1,8 +1,6 @@
 ---
 name: Slow CI
 on:
-  workflow_dispatch:
-  workflow_call:
   issue_comment:
     types: [created]
 env:
@@ -16,6 +14,12 @@ concurrency:
 jobs:
   check-membership:
     runs-on: ubuntu-latest
+    # if team member commented, not a draft, on a PR, using /fulltest
+    if: >
+      (github.event_name == 'issue_comment' && github.event.issue.pull_request &&
+      github.event.pull_request.draft == false && contains(github.event.comment.body,
+      '/fulltest')) ||
+      (github.event_name == 'workflow_dispatch')
     outputs:
       member_check: ${{ steps.is_member.outputs.member_check }}
     steps:
@@ -27,6 +31,19 @@ jobs:
           else
             echo "::set-output name=member_check::false"
           fi
+  comment-workflow-link:
+    runs-on: ubuntu-latest
+    if: (github.event_name == 'issue_comment' && github.event.issue.pull_request &&
+      github.event.pull_request.draft == false && contains(github.event.comment.body,
+      '/fulltest'))
+    steps:
+      - name: Post Comment
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            Triggered Slow CI: [Workflow Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          reactions: rocket
   docstring-check:
     needs: check-membership
     # if team member commented, not a draft, on a PR, using /fulltest
@@ -40,8 +57,26 @@ jobs:
       )
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/[email protected]
+      - uses: actions/github-script@v3
+        id: get-pr
+        with:
+          script: |
+            const request = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            }
+            core.info(`Getting PR  #${request.pull_number} from ${request.owner}/${request.repo}`)
+            try {
+              const result = await github.pulls.get(request)
+              return result.data
+            } catch (err) {
+              core.setFailed(`Request failed with error ${err}`)
+            }
+      - uses: actions/[email protected]
+        with:
+          repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+          ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }}
       - name: Set up Python
         uses: actions/[email protected]
         with:
@@ -63,8 +98,26 @@ jobs:
         (github.event_name == 'workflow_dispatch')
       )
     steps:
-      - name: Checkout code
-        uses: actions/[email protected]
+      - uses: actions/github-script@v3
+        id: get-pr
+        with:
+          script: |
+            const request = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            }
+            core.info(`Getting PR  #${request.pull_number} from ${request.owner}/${request.repo}`)
+            try {
+              const result = await github.pulls.get(request)
+              return result.data
+            } catch (err) {
+              core.setFailed(`Request failed with error ${err}`)
+            }
+      - uses: actions/[email protected]
+        with:
+          repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+          ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }}
       - name: Set up Python 3.9
         uses: actions/[email protected]
         with:
@@ -84,8 +137,26 @@ jobs:
       )
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/[email protected]
+      - uses: actions/github-script@v3
+        id: get-pr
+        with:
+          script: |
+            const request = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            }
+            core.info(`Getting PR  #${request.pull_number} from ${request.owner}/${request.repo}`)
+            try {
+              const result = await github.pulls.get(request)
+              return result.data
+            } catch (err) {
+              core.setFailed(`Request failed with error ${err}`)
+            }
+      - uses: actions/[email protected]
+        with:
+          repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+          ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }}
       - name: Set up Python
         uses: actions/[email protected]
         with:
@@ -290,3 +361,33 @@ jobs:
       test_environment: ${{ matrix.test_environment }}
       is-slow-ci: true
     secrets: inherit
+  post-ready-to-merge-comment:
+    runs-on: ubuntu-latest
+    needs:
+      - check-membership
+      - comment-workflow-link
+      - small-checks
+      - sqlite-db-migration-testing
+      - custom-ubuntu-unit-test
+      - windows-unit-test
+      - macos-unit-test
+      - windows-integration-test
+      - macos-integration-test
+      - custom-ubuntu-integration-test
+      - update-templates-to-examples
+      - docstring-check
+    # if team member commented, not a draft, on a PR, using /fulltest
+    if: >
+      (github.event_name == 'issue_comment' && github.event.issue.pull_request &&
+      github.event.pull_request.draft == false && contains(github.event.comment.body,
+      '/fulltest')) ||
+      (github.event_name == 'workflow_dispatch')
+    steps:
+      - name: Post Comment
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            🦭 All tests passed!
+            You are ready to merge this PR. 🚀
+          reactions: rocket

.github/workflows/generate-test-duration.yml

@@ -0,0 +1,56 @@
+---
+name: Generate test duration file
+on:
+  workflow_call:
+  schedule:
+    - cron: 0 8 * * 1  # Run every Monday at 8 am
+jobs:
+  generate-test-duration-file:
+    name: Generate test duration file
+    runs-on: ubuntu-dind-runners
+    strategy:
+      fail-fast: false
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: utf-8
+          # on MAC OS, we need to set this environment variable
+          # to fix problems with the fork() calls (see this thread
+          # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
+      OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/[email protected]
+        with:
+          ref: develop
+      - name: Setup environment
+        uses: ./.github/actions/setup_environment
+        with:
+          cache_version: ${{ secrets.GH_ACTIONS_CACHE_KEY }}
+          python-version: '3.10'
+          os: ubuntu-dind-runners
+          runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
+          runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
+      - name: Generate test duration file
+        continue-on-error: true
+            # Ubuntu integration tests run as 6 shards
+        run: |
+          bash scripts/test-coverage-xml.sh "" default "" "" store-durations
+      - name: Check difference in .test_durations
+        run: |
+          git diff --quiet || echo "Changes found"
+          if [ -n "$(git status --porcelain)" ]; then
+            # Commit changes directly to the specified branch
+            git add .test_durations
+            git commit -m "Update with new changes for test duration file at $(date +'%Y%m%d-%H%M%S')"
+            git push origin develop
+          else
+            echo "No changes in .test_durations"
+          fi
+      - name: Verify Python Env unaffected
+        run: |-
+          zenml integration list
+          pip list
+          pip check || true

.github/workflows/integration-test-slow.yml

@@ -129,7 +129,26 @@ jobs:
         if: inputs.os=='ubuntu-latest' && (contains(inputs.test_environment, 'docker')
           || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
           'airflow') || contains(inputs.test_environment, 'kubernetes'))
+      - uses: actions/github-script@v3
+        id: get-pr
+        with:
+          script: |
+            const request = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            }
+            core.info(`Getting PR  #${request.pull_number} from ${request.owner}/${request.repo}`)
+            try {
+              const result = await github.pulls.get(request)
+              return result.data
+            } catch (err) {
+              core.setFailed(`Request failed with error ${err}`)
+            }
       - uses: actions/[email protected]
+        with:
+          repository: ${{ fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+          ref: ${{ fromJSON(steps.get-pr.outputs.result).head.sha }}
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/publish_to_pypi.yml

@@ -6,6 +6,10 @@ jobs:
   publish_to_pypi:
     name: Publish Python 🐍 package 📦 to PyPI
     runs-on: ubuntu-latest
+    permissions:
+      # This permission is required for trusted publishing.
+      id-token: write
+      contents: read
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
@@ -26,12 +30,14 @@ jobs:
           virtualenvs-in-project: true
       - name: Include latest dashboard
         run: bash scripts/install-dashboard.sh
-      - name: publish
-        env:
-          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      - name: Build package
+        run: poetry build
+      - name: Mint token
+        id: mint
+        uses: tschm/[email protected]
+      - name: Publish the package with poetry
         run: |-
           if [ "$(cat src/zenml/VERSION)" = "$(echo ${GITHUB_REF} | sed 's|refs/tags/||g')" ];
-          then ./scripts/publish.sh;
+          then poetry publish -u __token__ -p '${{ steps.mint.outputs.api-token }}';
           else echo "Version mismatch between src/zenml/VERSION and branch tag" && exit 1;
           fi

.github/workflows/publish_to_pypi_nightly.yml

@@ -46,8 +46,14 @@ jobs:
           poetry version $NIGHTLY_VERSION
       - name: Include latest dashboard
         run: bash scripts/install-dashboard.sh
-      - name: publish
-        env:
-          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        run: bash scripts/publish.sh
+      - name: Build package
+        run: poetry build
+      - name: Mint token
+        id: mint
+        uses: tschm/[email protected]
+      - name: Publish the package with poetry
+        run: |-
+          if [ "$(cat src/zenml/VERSION)" = "$(echo ${GITHUB_REF} | sed 's|refs/tags/||g')" ];
+          then poetry publish -u __token__ -p '${{ steps.mint.outputs.api-token }}';
+          else echo "Version mismatch between src/zenml/VERSION and branch tag" && exit 1;
+          fi

.github/workflows/unit-test.yml

@@ -73,9 +73,29 @@ jobs:
       run:
         shell: bash
     steps:
-      - uses: actions/[email protected]
+      - name: Get PR details (for non-pull_request events)
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: actions/github-script@v3
+        id: get-pr
         with:
-          ref: ${{ github.event.inputs.git-ref || github.ref }}
+          script: |
+            const request = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            }
+            core.info(`Getting PR  #${request.pull_number} from ${request.owner}/${request.repo}`)
+            try {
+              const result = await github.pulls.get(request)
+              return result.data
+            } catch (err) {
+              core.setFailed(`Request failed with error ${err}`)
+            }
+      - name: Checkout code
+        uses: actions/[email protected]
+        with:
+          repository: ${{ github.event_name == 'pull_request' && github.repository || fromJSON(steps.get-pr.outputs.result).head.repo.full_name }}
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || fromJSON(steps.get-pr.outputs.result).head.sha }}
       - name: Setup environment
         uses: ./.github/actions/setup_environment
         with: