3461 push the image after building it ci (#3482)

Adding in the ability to push the image to Dockerhub after Earthly builds it. --------- Co-authored-by: Ross Jones <[email protected]>
bacalhau-project · Feb 26, 2024 · 212bdb4 · 212bdb4
1 parent 9b3ca23
commit 212bdb4
Show file tree

Hide file tree

Showing 442 changed files with 20,918 additions and 11,172 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -148,6 +148,7 @@ commands:
     steps:
       - restore_cache:
           keys:
+            - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "poetry.lock" }}
             - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
             - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}
             - asdf-<< parameters.cache_version >>-
@@ -165,7 +166,7 @@ commands:
             git checkout tags/<< parameters.asdf_version >>
             cd -
       - save_cache:
-          key: asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}
+          key: asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "poetry.lock" }}
           paths:
             - $HOME/.asdf
       - run:

diff --git a/.circleci/trigger_pipeline.sh b/.circleci/trigger_pipeline.sh
@@ -4,4 +4,4 @@ curl -X POST --header "Content-Type: application/json" --header "Circle-Token: $
     \"GHA_Action\": \"trigger_pipeline\"
  },
  \"branch\": \"${BRANCH}\"
-}" https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline
+}" https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline
diff --git a/.cspell-docs.json b/.cspell-docs.json
@@ -54,4 +54,4 @@
         "/ipfs://\\w*/",
         "/Qm[\\w]+[^\\w]/"
     ]
-}
+}
diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt
@@ -232,8 +232,8 @@ pbnode
 pflag
 phayes
 PKCS
-popd
 pmmmwh
+popd
 projectcontour
 proto
 protobuf
@@ -275,6 +275,7 @@ softmax
 sparkoperator
 stackdriver
 stretchr
+stringutils
 Strs
 struct
 svgr

diff --git a/.envrc b/.envrc
@@ -1,2 +1 @@
 source .venv/bin/activate
-
diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml
@@ -5,7 +5,10 @@ on:
   push:
     branches:
       - main
-
+  workflow_run:
+    workflows: ["Dependabot"]
+    types:
+      - completed
 
 permissions:
   contents: write

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -45,7 +45,7 @@ jobs:
       github.ref == 'refs/heads/main' &&
       !contains(github.event.pull_request.labels.*.name, 'do-not-merge') &&
       !contains(github.event.pull_request.labels.*.name, 'wip')
-    
+
 
     # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
     permissions:

diff --git a/.github/workflows/trigger-development-deployment.yaml b/.github/workflows/trigger-development-deployment.yaml
@@ -18,4 +18,4 @@ jobs:
           -H "Authorization: token ${{ secrets.BACALHAU_VMI_DEPLOYMENT_SECRET }}" \
           -H "Accept: application/vnd.github+json" \
           "https://api.github.com/repos/bacalhau-project/bacalhau-vmi/actions/workflows/deploy-commit.yaml/dispatches" \
-          --data '{"ref": "main", "inputs": {"commit_reference": "${{ github.sha }}", "redeploy": "false"}}'
+          --data '{"ref": "main", "inputs": {"commit_reference": "${{ github.sha }}", "redeploy": "false"}}'
diff --git a/.gitignore b/.gitignore
@@ -62,3 +62,5 @@ dist/
 
 .secret
 .arg
+
+.ruff_cache
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -11,4 +11,4 @@ tasks:
 
 vscode:
   extensions:
-    - hashicorp.terraform
+    - hashicorp.terraform
diff --git a/.gitprecommit/aws_key_checker.sh b/.gitprecommit/aws_key_checker.sh
@@ -11,7 +11,7 @@ fi
 
 # Redirect output to stderr.
 exec 1>&2
- 
+
 # Check changed files for an AWS keys
 FILES=$(git diff --cached --name-only "${against}")
 

diff --git a/.gitprecommit/go_fmt.sh b/.gitprecommit/go_fmt.sh
@@ -15,4 +15,4 @@ if [[ -n "${FILES}" ]]; then
             gofmt -w -s "${f}"
         fi
     done
-fi
+fi
diff --git a/DESIGN.md b/DESIGN.md
@@ -39,7 +39,7 @@ The flow of a submission will look like:
 ipfs job submit -x bafye9d4c615a2d4f7c4413cw5aec9c2f3308a6d32ffa -c bafy2bzacedcdedrghloawlwkntdhqnknqzxgh26ddwix7ld2a5ygagco3ngee
 ```
 
-One could imagine a veneer over the top of this that allows for submitting "linux compatible" things. For ex, the below would be wrapped in a 
+One could imagine a veneer over the top of this that allows for submitting "linux compatible" things. For ex, the below would be wrapped in a
 
 ```bash
 # PSEUDO CODE - returns all log file entries where GPS is within a range of a certain location
@@ -122,7 +122,7 @@ ifps job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrz
 - Need a dataset aggregator - where a single large dataset can describe many CIDs that may span sectors
 
 ## What's with the Name?
-Bacalhau means cod (the fish) in Portuguese (where several folks were brainstorming this topic). 
+Bacalhau means cod (the fish) in Portuguese (where several folks were brainstorming this topic).
 
 Compute-Over-Data == Cod == Bacalhau
 

diff --git a/LICENSE b/LICENSE
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.
diff --git a/README.md b/README.md
@@ -164,4 +164,4 @@ Others are allowed to make their own distribution of the software, but they cann
 
 We explicitly grant permission for you to make a build that includes our trademarks while developing Bacalhau software itself. You may not publish or share the build, and you may not use that build to run Bacalhau software for any other purpose.
 
-We have borrowed the above Open Source clause from the excellent [System Initiative](https://github.com/systeminit/si/blob/main/CONTRIBUTING.md) 
+We have borrowed the above Open Source clause from the excellent [System Initiative](https://github.com/systeminit/si/blob/main/CONTRIBUTING.md)
diff --git a/apps/job-info-consumer/Makefile b/apps/job-info-consumer/Makefile
@@ -48,4 +48,4 @@ docker-compose-push:
 
 .PHONY: restart
 restart:
-	gcloud compute ssh dashboard-vm-default-0 -- "cd ${DATA_DIR} && sudo docker-compose down && sudo docker-compose up -d"
+	gcloud compute ssh dashboard-vm-default-0 -- "cd ${DATA_DIR} && sudo docker-compose down && sudo docker-compose up -d"
diff --git a/apps/job-info-consumer/consumer/pkg/migrations/0001_consumer.down.sql b/apps/job-info-consumer/consumer/pkg/migrations/0001_consumer.down.sql
@@ -1,2 +1,2 @@
 DROP TABLE job_info;
-DROP VIEW job_summary_view
+DROP VIEW job_summary_view
diff --git a/apps/job-info-consumer/consumer/pkg/postgres.go b/apps/job-info-consumer/consumer/pkg/postgres.go
@@ -72,8 +72,8 @@ func (d *PostgresDatastore) InsertJobInfo(ctx context.Context, envelope jobinfo.
 
 	// Perform the insert or update using the ON CONFLICT DO UPDATE clause
 	_, err = d.db.Exec(`
-INSERT INTO job_info (id, apiversion, info) 
-VALUES ($1, $2, $3) 
+INSERT INTO job_info (id, apiversion, info)
+VALUES ($1, $2, $3)
 ON CONFLICT (id) DO UPDATE SET apiversion = $2, info = $3
 `, envelope.ID, envelope.APIVersion, infoJSON)
 	if err != nil {

diff --git a/apps/job-info-consumer/docker-compose.yml b/apps/job-info-consumer/docker-compose.yml
@@ -11,4 +11,4 @@ services:
     image: ${IMAGE_CONSUMER}
     restart: always
     depends_on:
-      - postgres
+      - postgres
diff --git a/apps/job-info-consumer/sample.env b/apps/job-info-consumer/sample.env
@@ -8,4 +8,4 @@ POSTGRES_DATA_DIR=/data/dashboard/postgres
 BACALHAU_PEER_CONNECT="/ip4/35.245.161.250/tcp/1235/p2p/QmbxGSsM6saCTyKkiWSxhJCt6Fgj7M9cns1vzYtfDbB5Ws,/ip4/35.245.61.251/tcp/1235/p2p/QmXaXu9N5GNetatsvwnTfQqNtSeKAD6uCmarbh3LMRYAcF,/ip4/35.245.251.239/tcp/1235/p2p/QmYgxZiySj3MRkwLSL4X2MF5F9f2PMhAE3LV49XkfNL1o3"
 IMAGE_CONSUMER=gcr.io/bacalhau-production/dashboard-consumer:latest
 
-POSTGRES_PASSWORD=<FILL_IN>
+POSTGRES_PASSWORD=<FILL_IN>
diff --git a/apps/job-info-consumer/terraform/main.tf b/apps/job-info-consumer/terraform/main.tf
@@ -152,7 +152,7 @@ resource "google_compute_firewall" "dashboard_ssh_firewall" {
 
   allow {
     protocol = "tcp"
-    // Port 22   - Provides ssh access to the bacalhau server, for debugging 
+    // Port 22   - Provides ssh access to the bacalhau server, for debugging
     ports = ["22"]
   }
 

diff --git a/apps/job-info-consumer/terraform/prod.tfvars b/apps/job-info-consumer/terraform/prod.tfvars
@@ -8,4 +8,4 @@ volume_size_gb             = 500
 boot_disk_size_gb          = 500
 ingress_cidrs              = ["0.0.0.0/0"]
 ssh_access_cidrs           = ["0.0.0.0/0"]
-postgres_access_cidrs      = ["0.0.0.0/0"]
+postgres_access_cidrs      = ["0.0.0.0/0"]
diff --git a/apps/job-info-consumer/terraform/remote_files/scripts/install-node.sh b/apps/job-info-consumer/terraform/remote_files/scripts/install-node.sh
@@ -20,15 +20,15 @@ function install-docker() {
   sudo chmod +x /usr/local/bin/docker-compose
 }
 
-function mount-disk() { 
+function mount-disk() {
   # wait for /dev/sdb to exist
   while [[ ! -e /dev/sdb ]]; do
     sleep 1
     echo "waiting for /dev/sdb to exist"
   done
   # mount /dev/sdb at /data
   sudo mkdir -p /data
-  sudo mount /dev/sdb /data || (sudo mkfs -t ext4 /dev/sdb && sudo mount /dev/sdb /data) 
+  sudo mount /dev/sdb /data || (sudo mkfs -t ext4 /dev/sdb && sudo mount /dev/sdb /data)
 }
 
 function install() {

diff --git a/apps/job-info-consumer/terraform/variables.tf b/apps/job-info-consumer/terraform/variables.tf
@@ -32,4 +32,4 @@ variable "ssh_access_cidrs" {
 variable "postgres_access_cidrs" {
   type    = set(string)
   default = []
-}
+}
diff --git a/benchmark/parse.py b/benchmark/parse.py
@@ -17,7 +17,6 @@
 # print(runs)
 
 for run in sorted(runs):
-
     exitCodes = defaultdict(int)
     means = []
 
@@ -38,7 +37,7 @@
                 exitCodes[code] += 1
             means.append(js["results"][0]["mean"])
         # trunk-ignore(flake8/E722)
-        except:
+        except Exception:
             pass
     print(f"    exitCodes: {dict(exitCodes)}")
     if means:

diff --git a/benchmark/quick_submit.sh b/benchmark/quick_submit.sh
@@ -4,4 +4,3 @@ while true; do
   printf %s\\n {0..1000} | xargs -n 1 -P 100 -i \
     "${BACALHAU_BIN}" --api-port="${API_PORT}" --api-host=localhost docker run --concurrency=3 busybox -- /bin/true
 done
-
diff --git a/benchmark/single_explode.sh b/benchmark/single_explode.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 export INTERATION_ID="$1"
 
 # select the port of the next requester node
-export API_PORT=$((${API_PORT:-"20000"}  + INTERATION_ID % REQUESTER_NODES)) 
+export API_PORT=$((${API_PORT:-"20000"}  + INTERATION_ID % REQUESTER_NODES))
 
 sudo tee "./results/parameters-${RUN_ID}.json" >/dev/null <<EOI
 {

diff --git a/clients/python/LICENSE b/clients/python/LICENSE
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.