Merge branch 'bianca-worker-guide' of https://github.com/biancaines/p…

…refect-gcp into bianca-worker-guide
PrefectHQ · Oct 3, 2023 · a3c5ea7 · a3c5ea7
2 parents 0ac25f6 + fdb0509
commit a3c5ea7
Show file tree

Hide file tree

Showing 27 changed files with 982 additions and 44 deletions.
diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml
@@ -11,7 +11,7 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: tibdex/github-app-token@v1
+      - uses: tibdex/github-app-token@v2
         id: generate-token
         name: Generate GitHub token
         with:

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

diff --git a/.github/workflows/nightly-dev-tests.yml b/.github/workflows/nightly-dev-tests.yml
@@ -17,7 +17,7 @@ jobs:
           - "3.10"
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4

diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v4

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,7 +10,7 @@ jobs:
     name: Build Release
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -57,7 +57,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v4

diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           persist-credentials: false
 

diff --git a/.github/workflows/template-sync.yml b/.github/workflows/template-sync.yml
@@ -9,7 +9,7 @@ jobs:
     name: Submit update PR
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -22,7 +22,7 @@ jobs:
       - name: Perform updates
         run: cruft update -y
 
-      - uses: tibdex/github-app-token@v1
+      - uses: tibdex/github-app-token@v2
         id: generate-token
         name: Generate GitHub token
         with:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -15,7 +15,7 @@ jobs:
           - "3.10"
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4

diff --git a/.github/workflows/tests_extras.yml b/.github/workflows/tests_extras.yml
@@ -15,7 +15,7 @@ jobs:
           - "aiplatform"
       fail-fast: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.9
         uses: actions/setup-python@v4

diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml
@@ -13,7 +13,7 @@ jobs:
           - "3.9"
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,15 +17,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fix `list_folders` and `list_blobs` now logging bucket name and bucket path - [#184](https://github.com/PrefectHQ/prefect-gcp/pull/214)
+
 ### Security
 
+## 0.4.7
+
+Released September 22nd, 2023.
+
+### Added
+
+- Vertex AI `CustomJob` worker - [#211](https://github.com/PrefectHQ/prefect-gcp/pull/211)
+- Add `kill_infrastructure` method to Vertex AI worker - [#213](https://github.com/PrefectHQ/prefect-gcp/pull/213)
+
+### Changed
+
+- Use flow run name for name of created custom jobs - [#208](https://github.com/PrefectHQ/prefect-gcp/pull/208)
+
 ## 0.4.6
 
-Not yet released
+Released September 5th, 2023.
 
 ### Changed
 
-- Vertex AI CustomJob sets labels specified by Prefect Agent when Deployment triggered on infrastructure.
+- Persist Labels to Vertex AI Custom Job - [#198](https://github.com/PrefectHQ/prefect-gcp/pull/208) 
 
 ## 0.4.5
 

diff --git a/docs/cloud_run_worker.md b/docs/cloud_run_worker.md
@@ -0,0 +1 @@
+::: prefect_gcp.workers.cloud_run
diff --git a/docs/vertex_worker.md b/docs/vertex_worker.md
@@ -0,0 +1 @@
+::: prefect_gcp.workers.vertex
diff --git a/docs/worker.md b/docs/worker.md
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -86,7 +86,9 @@ nav:
         - Cloud Run: cloud_run.md
         - AI Platform: aiplatform.md
         - Deployment Steps: deployments/steps.md
-        - Worker: worker.md
+        - Workers:
+          - Cloud Run: cloud_run_worker.md
+          - Vertex AI: vertex_worker.md
 
 extra:
     social:

diff --git a/prefect_gcp/__init__.py b/prefect_gcp/__init__.py
@@ -1,19 +1,23 @@
-from . import _version
+from prefect._internal.compatibility.deprecated import (
+    register_renamed_module,
+)
 
-from .bigquery import BigQueryWarehouse  # noqa
+from . import _version
 from .aiplatform import VertexAICustomTrainingJob  # noqa
-from .cloud_storage import GcsBucket  # noqa
+from .bigquery import BigQueryWarehouse  # noqa
 from .cloud_run import CloudRunJob  # noqa
-from .secret_manager import GcpSecret  # noqa
+from .cloud_storage import GcsBucket  # noqa
 from .credentials import GcpCredentials  # noqa
-from .worker import CloudRunWorker  # noqa
-from prefect._internal.compatibility.deprecated import (
-    register_renamed_module,
-)
+from .secret_manager import GcpSecret  # noqa
+from .workers.vertex import VertexAIWorker  # noqa
+from .workers.cloud_run import CloudRunWorker  # noqa
 
 register_renamed_module(
     "prefect_gcp.projects", "prefect_gcp.deployments", start_date="Jun 2023"
 )
+register_renamed_module(
+    "prefect_gcp.worker", "prefect_gcp.workers", start_date="Sep 2023"
+)
 
 
 __version__ = _version.get_versions()["version"]
diff --git a/prefect_gcp/aiplatform.py b/prefect_gcp/aiplatform.py
@@ -102,7 +102,7 @@ class VertexAICustomTrainingJob(Infrastructure):
 
     _block_type_name = "Vertex AI Custom Training Job"
     _block_type_slug = "vertex-ai-custom-training-job"
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/aiplatform/#prefect_gcp.aiplatform.VertexAICustomTrainingJob"  # noqa: E501
 
     type: Literal["vertex-ai-custom-training-job"] = Field(
@@ -184,7 +184,6 @@ class VertexAICustomTrainingJob(Infrastructure):
             "and required if a service account cannot be detected in gcp_credentials."
         ),
     )
-
     job_watch_poll_interval: float = Field(
         default=5.0,
         description=(
@@ -200,17 +199,14 @@ def job_name(self):
         https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.CustomJob#google_cloud_aiplatform_CustomJob_display_name
         """  # noqa
         try:
-            repo_name = self.image.split("/")[2]  # `gcr.io/<project_name>/<repo>/`"
+            base_name = self.name or self.image.split("/")[2]
+            return f"{base_name}-{uuid4().hex}"
         except IndexError:
             raise ValueError(
                 "The provided image must be from either Google Container Registry "
                 "or Google Artifact Registry"
             )
 
-        unique_suffix = uuid4().hex
-        job_name = f"{repo_name}-{unique_suffix}"
-        return job_name
-
     def _get_compatible_labels(self) -> Dict[str, str]:
         """
         Ensures labels are compatible with GCP label requirements.
@@ -430,6 +426,7 @@ async def run(
             raise RuntimeError(f"{self._log_prefix}: {error_msg}")
 
         status_code = 0 if final_job_run.state == JobState.JOB_STATE_SUCCEEDED else 1
+
         return VertexAICustomTrainingJobResult(
             identifier=final_job_run.display_name, status_code=status_code
         )

diff --git a/prefect_gcp/bigquery.py b/prefect_gcp/bigquery.py
@@ -552,7 +552,7 @@ class BigQueryWarehouse(DatabaseBlock):
     """  # noqa
 
     _block_type_name = "BigQuery Warehouse"
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/bigquery/#prefect_gcp.bigquery.BigQueryWarehouse"  # noqa: E501
 
     gcp_credentials: GcpCredentials

diff --git a/prefect_gcp/cloud_run.py b/prefect_gcp/cloud_run.py
@@ -213,7 +213,7 @@ class CloudRunJob(Infrastructure):
     _block_type_slug = "cloud-run-job"
     _block_type_name = "GCP Cloud Run Job"
     _description = "Infrastructure block used to run GCP Cloud Run Jobs. Note this block is experimental. The interface may change without notice."  # noqa
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/cloud_run/#prefect_gcp.cloud_run.CloudRunJob"  # noqa: E501
 
     type: Literal["cloud-run-job"] = Field(

diff --git a/prefect_gcp/cloud_storage.py b/prefect_gcp/cloud_storage.py
@@ -564,7 +564,7 @@ class GcsBucket(WritableDeploymentStorage, WritableFileSystem, ObjectStorageBloc
         ```
     """
 
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _block_type_name = "GCS Bucket"
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/cloud_storage/#prefect_gcp.cloud_storage.GcsBucket"  # noqa: E501
 
@@ -870,7 +870,12 @@ async def list_blobs(self, folder: str = "") -> List["Blob"]:
         client = self.gcp_credentials.get_cloud_storage_client()
 
         bucket_path = self._join_bucket_folder(folder)
-        self.logger.info(f"Listing blobs in bucket {bucket_path}.")
+        if bucket_path is None:
+            self.logger.info(f"Listing blobs in bucket {self.bucket!r}.")
+        else:
+            self.logger.info(
+                f"Listing blobs in folder {bucket_path!r} in bucket {self.bucket!r}."
+            )
         blobs = await run_sync_in_worker_thread(
             client.list_blobs, self.bucket, prefix=bucket_path
         )
@@ -903,12 +908,19 @@ async def list_folders(self, folder: str = "") -> List[str]:
             from prefect_gcp.cloud_storage import GcsBucket
 
             gcs_bucket = GcsBucket.load("my-bucket")
-            gcs_bucket.list_folders('years)
+            gcs_bucket.list_folders("years")
             ```
         """
 
-        bucket_path = self._join_bucket_folder()
-        self.logger.info(f"Listing folders in bucket {bucket_path}.")
+        # Beware of calling _join_bucket_folder twice, see note in method.
+        # However, we just want to use it to check if we are listing the root folder
+        bucket_path = self._join_bucket_folder(folder)
+        if bucket_path is None:
+            self.logger.info(f"Listing folders in bucket {self.bucket!r}.")
+        else:
+            self.logger.info(
+                f"Listing folders in {bucket_path!r} in bucket {self.bucket!r}."
+            )
 
         blobs = await self.list_blobs(folder)
         # gets all folders with full path

diff --git a/prefect_gcp/credentials.py b/prefect_gcp/credentials.py
@@ -96,7 +96,7 @@ class GcpCredentials(CredentialsBlock):
         ```
     """  # noqa
 
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _block_type_name = "GCP Credentials"
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/credentials/#prefect_gcp.credentials.GcpCredentials"  # noqa: E501
 

diff --git a/prefect_gcp/secret_manager.py b/prefect_gcp/secret_manager.py
@@ -300,7 +300,7 @@ class GcpSecret(SecretBlock):
         secret_version: Version number of the secret to use, or "latest".
     """
 
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4CD4wwbiIKPkZDt4U3TEuW/c112fe85653da054b6d5334ef662bec4/gcp.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
     _documentation_url = "https://prefecthq.github.io/prefect-gcp/secret_manager/#prefect_gcp.secret_manager.GcpSecret"  # noqa: E501
 
     gcp_credentials: GcpCredentials

diff --git a/prefect_gcp/workers/__init__.py b/prefect_gcp/workers/__init__.py
diff --git a/prefect_gcp/worker.py → prefect_gcp/workers/cloud_run.py b/prefect_gcp/worker.py → prefect_gcp/workers/cloud_run.py
@@ -519,8 +519,8 @@ class CloudRunWorker(BaseWorker):
         "a Google Cloud Platform account."
     )
     _display_name = "Google Cloud Run"
-    _documentation_url = "https://prefecthq.github.io/prefect-gcp/worker/"
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/4SpnOBvMYkHp6z939MDKP6/549a91bc1ce9afd4fb12c68db7b68106/social-icon-google-cloud-1200-630.png?h=250"  # noqa
+    _documentation_url = "https://prefecthq.github.io/prefect-gcp/cloud_run_worker/"
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png"  # noqa
 
     def _create_job_error(self, exc, configuration):
         """Provides a nicer error for 404s when trying to create a Cloud Run Job."""