Bump github.com/aws/aws-sdk-go-v2 from 1.21.1 to 1.24.1 (#3405)

Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.21.1 to 1.24.1. <details> <summary>Commits</summary> <ul> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/47dd1b1bcbde244357a82ef00fa6a61a9bfb9b39"><code>47dd1b1</code></a> Release 2024-01-04</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/23145e3e605a93582020facfe13350b4153714e1"><code>23145e3</code></a> Regenerated Clients</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/00e57bb7feb2d104387103aa4fbcd55dfff3a6a7"><code>00e57bb</code></a> Update API model</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/73e1a99f2fa858ca56627779852768a9198ba057"><code>73e1a99</code></a> Merge pull request <a href="https://redirect.github.com/aws/aws-sdk-go-v2/issues/2441">#2441</a> from RanVaknin/fix-documentation-config</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/0f8ad11593c219c52ad7fb12998c75ade39dc7ad"><code>0f8ad11</code></a> Fix SRA auth trailing checksum retry bug (<a href="https://redirect.github.com/aws/aws-sdk-go-v2/issues/2438">#2438</a>)</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/efbc5aa622a882167129e69a88aa50c730cd1904"><code>efbc5aa</code></a> Release 2024-01-03</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/78357bb87682230e24b15c01e807d7375a9474e4"><code>78357bb</code></a> Regenerated Clients</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/e465ddd60d18e91b34de5917534cfa1542323027"><code>e465ddd</code></a> Update endpoints model</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/384ad3b7ec21eecb3c6c38b69f86fb6342906b11"><code>384ad3b</code></a> Update API model</li> <li><a href="https://github.com/aws/aws-sdk-go-v2/commit/1126a91e53a20b18bc1db74225a8417bfb63f427"><code>1126a91</code></a> changelog added</li> <li>Additional commits viewable in <a href="https://github.com/aws/aws-sdk-go-v2/compare/v1.21.1...v1.24.1">compare view</a></li> </ul> </details> <br /> [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go-v2&package-manager=go_modules&previous-version=1.21.1&new-version=1.24.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) --- <details> <summary>Dependabot commands and options</summary> <br /> You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show <dependency name> ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) </details> --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Aronchick <[email protected]>
bacalhau-project · Feb 13, 2024 · b58e8c8 · b58e8c8
1 parent fc34f69
commit b58e8c8
Show file tree

Hide file tree

Showing 20 changed files with 1,037 additions and 749 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,6 +1,8 @@
 # Use the latest 2.1 version of CircleCI pipeline process engine.
 # See: https://circleci.com/docs/2.0/configuration-reference
 version: 2.1
+# In order to call CircleCI with webhook, you need to set the parameters
+# and leave the default values empty.
 parameters:
   GHA_Actor:
     type: string
@@ -34,11 +36,11 @@ executors:
   linux-amd64:
     machine:
       image: default
-      resource_class: xlarge
+    resource_class: xlarge
   linux-arm64:
     machine:
       image: default
-      resource_class: arm.xlarge
+    resource_class: arm.xlarge
   darwin-amd64:
     macos:
       xcode: "14.2.0"
@@ -49,9 +51,12 @@ executors:
     resource_class: macos.m1.medium.gen1
   windows-amd64:
     machine:
-      image: windows-server-2022-gui:current
-      resource_class: windows.xlarge
-      shell: bash.exe
+      image: "windows-server-2022-gui:current"
+    resource_class: windows.xlarge
+  docker-linux:
+    docker:
+      - image: cimg/base:2022.01
+    resource_class: xlarge
 
 commands:
   install_go:
@@ -83,6 +88,7 @@ commands:
           steps:
             - run:
                 name: Install Go
+                shell: bash.exe
                 command: |
                   rm -rf /c/Program\ Files/Go
                   curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.zip | tar --extract --gzip --file=- --directory=/c/Program\ Files
@@ -125,12 +131,114 @@ commands:
             - ~/.cache/go-build
             - ~/go/pkg/mod
 
+  # Installs asdf, a version manager for golang, python, nodejs, etc. Installs from .tool-versions
+  install_asdf:
+    parameters:
+      cache_version:
+        type: string
+        default: v1-1
+      asdf_version:
+        type: string
+        default: v0.14.0
+    description: Install asdf and dependencies.
+    steps:
+      - restore_cache:
+          keys:
+            - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
+            - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}
+            - asdf-<< parameters.cache_version >>-
+      - run:
+          name: Install asdf
+          command: |
+            if [ ! -d $HOME/.asdf ]; then
+              git clone https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch << parameters.asdf_version >>
+              cd $HOME/.asdf
+            else
+              cd $HOME/.asdf
+              git fetch origin
+              git pull
+            fi
+            git checkout tags/<< parameters.asdf_version >>
+            cd -
+      - save_cache:
+          key: asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
+          paths:
+            - $HOME/.asdf
+      - run:
+          name: Set PATH
+          command: |
+            echo 'export PATH=$HOME/.asdf/bin:$HOME/.asdf/shims:"${PATH}"' >> "${BASH_ENV}"
+            source "${BASH_ENV}"
+
+  # Installs and caches python and dependencies. Installs from .tool-versions in the local directory
+  # which is a file that asdf uses to install the correct version of python.
   install_python:
+    description: Install Python and dependencies.
+    parameters:
+      cache_version:
+        type: string
+        default: v1-1
     steps:
-      - run: pyenv install 3.10
-      - run: pyenv global 3.10
-      - run: curl -sSL https://install.python-poetry.org | python3 -
-      - run: echo 'export PATH="/home/circleci/.local/bin:$PATH"' >> "$BASH_ENV"
+      - install_asdf:
+          cache_version: << parameters.cache_version >>
+      - restore_cache:
+          keys:
+            - python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
+            - python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}
+            - python-deps-<< parameters.cache_version >>-
+      - run:
+          name: Install toolchain from .tool-versions
+          command: |
+            asdf plugin-add python
+            asdf install python $(awk '/^python/ {print $2}' .tool-versions)
+            pip3 install --upgrade pip
+            pip3 install poetry # Install poetry in the newly created virtual environment
+            python -m poetry install
+      - run:
+          name: Install Python API client pre-requisites
+          command: |
+            CODEGEN_BASE_URL="https://repo1.maven.org/maven2/io/swagger/codegen/v3"
+            wget ${CODEGEN_BASE_URL}/swagger-codegen-cli/3.0.52/swagger-codegen-cli-3.0.52.jar -O ${HOME}/bin/swagger-codegen-cli.jar
+            chmod +x ${HOME}/bin/swagger-codegen-cli.jar
+            ${HOME}/bin/swagger-codegen-cli.jar version
+      - save_cache:
+          key: python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
+          paths:
+            - $HOME/.asdf
+            - $HOME/.cache/pip
+            - $HOME/.cache/pypoetry
+            - $HOME/.local
+      - run:
+          name: Set PATH
+          # Sources PATH after installing asdf and python, from $BASH_ENV, which CircleCI uses to set the PATH
+          command: echo 'export PATH="${PATH}":$HOME/.asdf/bin:$HOME/.asdf/shims:$HOME/.brew/bin' >> "${BASH_ENV}"
+
+  # When running the makefile, we need to ensure that we are using the correct shell
+  # so that the right shims are used. We cannot use global versions of tools because
+  # the makefile is not aware of the asdf environment.
+  run_make_under_specific_shell:
+    parameters:
+      directory_to_change_to:
+        type: string
+      makefile_directory:
+        type: string
+      make_command:
+        type: string
+    steps:
+      - run:
+          name: Run make for << parameters.make_command >> in << parameters.directory_to_change_to >> against make file in << parameters.makefile_directory >>
+          command: |
+            source $BASH_ENV
+            pushd << parameters.directory_to_change_to >>
+            python -m poetry install
+            source $(python -m poetry env info --path)/bin/activate # Doing this so it doesn't create a new shell
+            pip install poetry # Installing inside the virtual env
+            poetry install # Installing all the dependencies
+            make << parameters.make_command >> -C << parameters.makefile_directory >>
+            deactivate
+            # Return to the original directory
+            popd
+
 jobs:
   build_cli:
     executor: linux-amd64
@@ -279,57 +387,45 @@ jobs:
       - run:
           name: Copy swagger.json from workspace
           command: |
-            cp   /tmp/swagger_spec/webui/public/swagger/swagger.json ./docs/swagger.json
-      - run:
-          name: Install Python API client pre-requisites
-          command: |
-            CODEGEN_BASE_URL="https://repo1.maven.org/maven2/io/swagger/codegen/v3"
-            wget ${CODEGEN_BASE_URL}/swagger-codegen-cli/3.0.41/swagger-codegen-cli-3.0.41.jar -O ${HOME}/bin/swagger-codegen-cli.jar
-            chmod +x ${HOME}/bin/swagger-codegen-cli.jar
-            ${HOME}/bin/swagger-codegen-cli.jar version
+            cp /tmp/swagger_spec/webui/public/swagger/swagger.json ./docs/swagger.json
       - run:
           name: Build Python API client
-          command: |
-            make build-python-apiclient
+          command: make build-python-apiclient
       - persist_to_workspace:
           root: clients/python/dist
           paths:
             - bacalhau_apiclient-*.tar.gz
             - bacalhau_apiclient-*.whl
-      - run:
-          name: Install Python SDK pre-requisites
-          command: |
-            cd python
-            poetry run pip3 install black>=22.12.0 isort>=5.8.0 flake8>=6.0.0 mypy>=0.991 pre-commit>=2.12.0
-      - run:
-          name: Build Python SDK
-          command: |
-            make build-python-sdk
+      - run_make_under_specific_shell:
+          directory_to_change_to: $HOME/repo/python
+          makefile_directory: $HOME/repo
+          make_command: build-python-sdk
       - persist_to_workspace:
           root: python/dist
           paths:
             - bacalhau_sdk-*.tar.gz
             - bacalhau_sdk-*.whl
       - run:
           name: Install bacalhau-airflow pre-requisites
+          working_directory: integration/airflow
           command: |
-            cd integration/airflow
-            pip3 install -r dev-requirements.txt
-      - run:
-          name: Build Python Bacalhau Airflow
-          command: |
-            make build-bacalhau-airflow
-      - run:
-          name: Install Flyte plugin pre-requisites
-          command: |
-            cd integration/flyte
-            # installs flytekit, flyteidl and deps.
-            make setup
-      - run:
-          name: Build Python Bacalhau Flyte plugin
-          command: |
-            cd integration/flyte/plugins/
-            make build_all_plugins
+            source "$BASH_ENV"
+            python -m poetry run pip install -r dev-requirements.txt
+      - run_make_under_specific_shell:
+          # Build the airflow integration
+          directory_to_change_to: $HOME/repo/integration/airflow
+          makefile_directory: $HOME/repo
+          make_command: build-bacalhau-airflow
+      - run_make_under_specific_shell:
+          # Setup flyte integration
+          directory_to_change_to: $HOME/repo/integration/flyte
+          makefile_directory: $HOME/repo/integration/flyte
+          make_command: setup
+      - run_make_under_specific_shell:
+          # Build the flyte integration
+          directory_to_change_to: $HOME/repo/integration/flyte/plugins
+          makefile_directory: $HOME/repo/integration/flyte/plugins
+          make_command: build_all_plugins
       - persist_to_workspace:
           root: integration/airflow/dist
           paths:
@@ -408,23 +504,20 @@ jobs:
           paths:
             - "*.coverage"
 
-  build_python_sdk:
-    executor: linux-amd64
-    steps:
-      - checkout
-      - install_python
-      - run:
-          working_directory: python
-          command: make setup test
-
-  test_python_sdk:
+  build_and_test_python_sdk:
     executor: linux-amd64
+    working_directory: ~/repo
     steps:
       - checkout
       - install_python
       - run:
-          working_directory: python
-          command: make setup test
+          command: |
+            cd webui
+            yarn install && yarn run build
+      - run_make_under_specific_shell:
+          directory_to_change_to: $HOME/repo/python
+          makefile_directory: $HOME/repo/python
+          make_command: setup test
 
   test_python_airflow:
     executor: linux-amd64
@@ -433,7 +526,7 @@ jobs:
       - install_python
       - run:
           working_directory: integration/airflow
-          command: make setup test-all
+          command: python -m poetry run make setup test-all
 
   test_python_flyte:
     executor: linux-amd64
@@ -442,7 +535,7 @@ jobs:
       - install_python
       - run:
           working_directory: integration/flyte
-          command: make setup test
+          command: python -m poetry run make setup test
 
   coverage:
     executor: linux-amd64
@@ -502,24 +595,26 @@ jobs:
             cp /tmp/py_dist/bacalhau_sdk* python/dist
             mkdir -p integration/airflow/dist
             cp /tmp/py_dist/bacalhau_airflow* integration/airflow/dist
+            mkdir -p integration/airflow/dist
+            cp /tmp/py_dist/bacalhau_flyte* integration/flyte/dist
       - run:
           name: Release python apiclient
           command: |
-            make release-python-apiclient
+            python -m poetry run make release-python-apiclient
       - run:
           name: Release python sdk
           command: |
-            make release-python-sdk
+            python -m poetry run make release-python-sdk
       - run:
           name: Release python Airflow integration
           command: |
-            pip3 install twine==4.0.2
-            make release-bacalhau-airflow
+            python -m poetry run pip install twine==4.0.2
+            python -m poetry run make release-bacalhau-airflow
       - run:
           name: Release python Flyte integration
           command: |
-            pip3 install twine==4.0.2
-            make release-bacalhau-flyte
+            python -m poetry run pip install twine==4.0.2
+            python -m poetry run make release-bacalhau-flyte
 
   build_swagger:
     executor: linux-amd64
@@ -618,7 +713,7 @@ workflows:
               - target_os: windows
                 target_arch: arm64
                 build_tags: integration
-      - test_python_sdk
+      - build_and_test_python_sdk
       - test_python_airflow
       - test_python_flyte
       - coverage:
@@ -650,7 +745,7 @@ workflows:
           filters: &build_on_release_only
             branches:
               ignore: /.*/ # don't run on any branches - only tags
-            !!merge <<: *build_on_branch_and_tag_push
+            <<: *build_on_branch_and_tag_push
 
   build_workflow:
     when: << pipeline.parameters.GHA_Action >>