diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f5009edbe..291ab6decd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,18 +32,6 @@ parameters: default: "build-vm-linux-amd64" type: string - PUSH_BRANCH: - default: "main" - type: string - - PUSH_TAG: - default: "v0.0.0" - type: string - - full_name: - default: "bacalhau-project/bacalhau" - type: string - executors: # Available resource classes - https://circleci.com/product/features/resource-classes/ linux-amd64: diff --git a/.circleci/push-to-upstream-fork.py b/.circleci/push-to-upstream-fork.py deleted file mode 100644 index 0dfe9e05f9..0000000000 --- a/.circleci/push-to-upstream-fork.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import random -import string -import subprocess -import sys - -from github import Github, GithubException - - -def run_command(command): - result = subprocess.run( - command, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if result.returncode != 0: - print(f"Error: {result.stderr.strip()}", file=sys.stderr) - sys.exit(result.returncode) - return result.stdout.strip() - - -def remote_exists(remote_name): - result = subprocess.run( - f"git remote get-url {remote_name}", - shell=True, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return result.returncode == 0 - - -def get_pull_request_info(github_token, upstream_repo, source_gh_user, source_branch): - g = Github(github_token) - repo = g.get_repo(f"{upstream_repo}") - pull_requests = repo.get_pulls( - state="open", head=f"{source_gh_user}:{source_branch}" - ) - - if pull_requests.totalCount == 0: - print( - f"No open pull request found for branch {source_branch} in repo {upstream_repo}" - ) - sys.exit(1) - - pr = pull_requests[0] - return pr.title, pr.body - - -def create_pull_request( - github_token, upstream_repo, source_branch, new_branch_name, title, body -): - g = Github(github_token) - repo = g.get_repo(upstream_repo) - - try: - # Ensure the new branch exists before creating a pull request - repo.get_branch(new_branch_name) - except GithubException: - print(f"Error: Branch {new_branch_name} does not exist in the repository.") - sys.exit(1) - - if body is None: - body = "" - - try: - pr = repo.create_pull( - title=title, body=body, head=new_branch_name, base="main", draft=False - ) - print(f"Pull request created: {pr.html_url}") - except GithubException as e: - print(f"Failed to create pull request: {e}") - raise - - -def main(): - parser = argparse.ArgumentParser( - description="Push a forked branch to an upstream repository." - ) - parser.add_argument( - "--upstream_full_name", - type=str, - required=True, - help="The upstream full name (e.g.: '/' with the '/')", - ) - parser.add_argument( - "--branch_spec", - type=str, - required=True, - help="The fork username and branch name in the format :", - ) - parser.add_argument( - "--gpf_upstream_branch", - type=str, - required=True, - help="The upstream branch name", - ) - parser.add_argument( - "--github_token", type=str, required=True, help="GitHub token for API access" - ) - args = parser.parse_args() - - upstream_full_name = args.upstream_full_name - branch_spec = args.branch_spec - gpf_upstream_branch = args.gpf_upstream_branch - github_token = args.github_token - - if branch_spec.count(":") != 1: - parser.error( - "branch_spec must be in the format :" - ) - - source_gh_user, source_branch = branch_spec.split(":") - repo_name = run_command( - "git remote get-url --push origin | awk -F/ '{print $NF}' | sed 's/\\.git$//'" - ) - - random_string = "".join(random.choices(string.ascii_letters + string.digits, k=6)) - fork_to_test_name = f"fork-to-test-{random_string}" - - if remote_exists(fork_to_test_name): - run_command(f"git remote remove {fork_to_test_name}") - - gpf_use_ssh = os.getenv("GPF_USE_SSH", "") - - if gpf_use_ssh: - run_command( - f"git remote add {fork_to_test_name} git@github.com:{source_gh_user}/{repo_name}.git" - ) - else: - run_command( - f"git remote add {fork_to_test_name} https://github.com/{source_gh_user}/{repo_name}.git" - ) - - run_command("git fetch --all") - - new_branch_name = f"{gpf_upstream_branch}-pr-{random_string}" - run_command( - f"git push --force https://github.com/{upstream_full_name} refs/remotes/{fork_to_test_name}/{source_branch}:refs/heads/{new_branch_name}" - ) - - run_command(f"git remote remove {fork_to_test_name}") - - title, body = get_pull_request_info( - github_token, upstream_full_name, source_gh_user, source_branch - ) - create_pull_request( - github_token, upstream_full_name, source_branch, new_branch_name, title, body - ) - - print( - f"Forked branch '{branch_spec}' has been pushed to {upstream_full_name} branch '{new_branch_name}' and a pull request has been created." - ) - - -if __name__ == "__main__": - main() diff --git a/.circleci/trigger_circleci.py b/.circleci/trigger_circleci.py deleted file mode 100644 index 9b76d1cee6..0000000000 --- a/.circleci/trigger_circleci.py +++ /dev/null @@ -1,91 +0,0 @@ -import argparse -import json -import os -from pathlib import Path - -import requests -from dotenv import load_dotenv - - -def main(): - ref = os.getenv("REF") - circle_token = os.getenv("CIRCLE_TOKEN") - full_name = os.getenv("FULL_NAME") - - if not circle_token: - print("CIRCLE_TOKEN is not set. Exiting.") - exit(1) - - print(f"Ref: {ref}") - print(f"Full Name: {full_name}") - - if not ref: - target = {"branch": "main"} - elif "refs/tags" in ref: - tag = ref.replace("refs/tags/", "") - target = {"tag": tag} - else: - target = {"branch": ref} - - headers = { - "Content-Type": "application/json", - "Circle-Token": circle_token, - } - - data = { - "parameters": { - "GHA_Action": "trigger_pipeline", - "full_name": full_name, - }, - } - data.update(target) - - print(f"Full data: {data}") - - response = requests.post( - "https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline", - headers=headers, - data=json.dumps(data), - ) - - # If response code not in 2xx, raise an exception - if response.status_code < 200 or response.status_code >= 300: - print(f"Failed to trigger CircleCI pipeline: {response.status_code}") - print(response.text) - response.raise_for_status() - else: - print("Successfully triggered CircleCI pipeline") - - -if __name__ == "__main__": - # Get .env file as flag - argsp = argparse.ArgumentParser() - argsp.add_argument( - "--env", type=str, default=".env", required=False, help="Path to .env file." - ) - argsp.add_argument("--test", type=bool, default=False, help="Test mode.") - args = argsp.parse_args() - - if args.env is not None and args.env != "": - if Path(args.env).exists(): - load_dotenv(args.env) - else: - print(f"File {args.env} does not exist. Exiting.") - - if args.test: - if Path(args.env).exists(): - load_dotenv(args.env) - else: - print(f"File {args.env} does not exist. Exiting.") - exit - - os.environ["REF"] = "main" - os.environ["CIRCLE_TOKEN"] = os.environ["CIRCLE_TOKEN"] - os.environ["FULL_NAME"] = "aronchick/main" - - print("Running in test mode.") - print(f"REF: {os.getenv('REF')}") - print(f"Circle Token: {os.getenv('CIRCLE_TOKEN')}") - print(f"Full Name: {os.getenv('FULL_NAME')}") - - main() diff --git a/.circleci/trigger_pipeline.sh b/.circleci/trigger_pipeline.sh new file mode 100755 index 0000000000..172d1e4d53 --- /dev/null +++ b/.circleci/trigger_pipeline.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -xeuo pipefail +IFS=$'\n\t' + +if [[ -z "${BRANCH}" ]]; then + TARGET="\"branch\": \"main\"" + elif [[ "${BRANCH}" =~ "refs/tags" ]]; then + TAG=$(echo "${BRANCH}" | sed 's:refs/tags/::') + TARGET="\"tag\": \"${TAG}\"" +else + TARGET="\"branch\": \"${BRANCH}\"" +fi + +curl --fail -X POST --header "Content-Type: application/json" --header "Circle-Token: ${CIRCLE_TOKEN}" -d "{ + \"parameters\": { + \"GHA_Action\": \"trigger_pipeline\" + }, + ${TARGET} +}" https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline diff --git a/.github/trigger-on-label.yml b/.github/trigger-on-label.yml new file mode 100644 index 0000000000..405f507fa4 --- /dev/null +++ b/.github/trigger-on-label.yml @@ -0,0 +1,34 @@ +name: Trigger on Label + +on: + pull_request: + types: [labeled] + +jobs: + trigger-workflows: + if: github.event.label.name == 'ok-to-test' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install yq + run: sudo snap install yq + + - name: Check if labeler is authorized + id: check-authorization + run: | + LABELER=$(jq --raw-output .sender.login "$GITHUB_EVENT_PATH") + APPROVERS=$(yq e '.approvers[]' OWNERS) + if echo "$APPROVERS" | grep -Fxq "$LABELER"; then + echo "::set-output name=authorized::true" + else + echo "::set-output name=authorized::false" + fi + + - name: Dispatch event + if: steps.check-authorization.outputs.authorized == 'true' + uses: peter-evans/repository-dispatch@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + event-type: ok-to-test diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index bf9e1ce20b..ebc945d482 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -12,7 +12,7 @@ on: types: - completed repository_dispatch: - types: [ok-to-test, ok-to-trigger-circleci] + types: [ok-to-test] permissions: contents: write @@ -21,14 +21,12 @@ permissions: jobs: trigger-circle-ci: if: | - github.event_name != 'pull_request' || - ( - !contains(github.event.pull_request.labels.*.name, 'do-not-merge') && - !contains(github.event.pull_request.labels.*.name, 'wip') - ) + !contains(github.event.pull_request.labels.*.name, 'do-not-merge') && + !contains(github.event.pull_request.labels.*.name, 'wip') runs-on: ubuntu-latest name: Pull Request Triggered CircleCI steps: + # For Debugging Below - name: Dump GitHub context env: GITHUB_CONTEXT: ${{ toJson(github) }} @@ -46,41 +44,14 @@ jobs: RUNNER_CONTEXT: ${{ toJson(runner) }} run: echo "$RUNNER_CONTEXT" - uses: actions/checkout@v3 - - if: ${{ github.event_name != 'pull_request' }} - run: echo "Not a pull request. Exiting." - - name: Set up ref - id: set-ref + - name: Trigger CircleCI + id: trigger-circle-ci run: | - if [[ -n "${{ github.event.client_payload.ref }}" ]]; then - echo "Payload ref: ${{ github.event.client_payload.ref }}" - echo "REF=${{ github.event.client_payload.ref }}" >> $GITHUB_ENV - elif [[ -n "${{ github.event.pull_request.head.ref }}" ]]; then - echo "PR ref: ${{ github.event.pull_request.head.ref }}" - echo "REF=${{ github.event.pull_request.head.ref }}" >> $GITHUB_ENV - elif [[ -n "${{ github.sha }}" ]]; then - echo "REF: ${{ github.sha }}" - echo "REF=${{ github.sha }}" >> $GITHUB_ENV - else - echo "No SHA set. Exiting." + if [[ -z $CIRCLE_TOKEN ]]; then + echo "CIRCLE_TOKEN is not set. Exiting." exit 1 fi - - if [[ -n "${{ github.event.client_payload.full_name }}" ]]; then - echo "FULL_NAME=${{ github.event.client_payload.full_name }}" >> $GITHUB_ENV - else - echo "FULL_NAME=${{ github.repository }}" >> $GITHUB_ENV - fi - - - uses: hmarr/debug-action@v3 - - - name: "Trigger CircleCI" - id: trigger-circle-ci - run: | - python -m pip install poetry - poetry install - source $(poetry env info --path)/bin/activate - python ./.circleci/trigger_circleci.py + ./.circleci/trigger_pipeline.sh env: CIRCLE_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} - REF: ${{ env.REF }} - FULL_NAME: ${{ github.repository }} + BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event.ref }} diff --git a/.github/workflows/trigger-ga-vmi-pipeline-on-release.yml b/.github/workflows/trigger-ga-vmi-pipeline-on-release.yml index ed530897d3..65bbd237ac 100644 --- a/.github/workflows/trigger-ga-vmi-pipeline-on-release.yml +++ b/.github/workflows/trigger-ga-vmi-pipeline-on-release.yml @@ -4,6 +4,9 @@ on: release: types: [created] +permissions: + contents: write + jobs: handle-release: runs-on: ubuntu-latest @@ -15,8 +18,8 @@ jobs: env: REPO: ${{ github.repository }} run: | - curl -X POST \ - -H "Authorization: token ${{ secrets.BACALHAU_VMI_DEPLOYMENT_SECRET }}" \ + curl -L -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/$REPO/dispatches" \ -d '{"event_type": "new-version-release", "client_payload": {"release_tag": "${{ github.ref_name }}"} }' diff --git a/.github/workflows/trigger-ok-to-push-to-upstream.yaml b/.github/workflows/trigger-ok-to-push-to-upstream.yaml deleted file mode 100644 index 0b1e917b35..0000000000 --- a/.github/workflows/trigger-ok-to-push-to-upstream.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Ok to push to upstream - -on: - repository_dispatch: - types: [ok-to-push-to-upstream] - -permissions: - contents: write - pull-requests: write - -jobs: - trigger-push-to-upstream: - runs-on: ubuntu-latest - name: Push to upstream - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - uses: hmarr/debug-action@v3 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "pip" - - name: "Trigger Push to Upstream" - id: trigger-push-to-upstream-step - run: | - pip install PyGithub - - python ./.circleci/push-to-upstream-fork.py \ - --upstream_full_name ${{ github.event.client_payload.upstream_full_name }} \ - --upstream_branch ${{ github.event.client_payload.upstream_branch }} \ - --branch_spec ${{ github.event.client_payload.user }}/${{ github.event.client_payload.branch }} \ - --github_token ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/trigger-on-comment.yml b/.github/workflows/trigger-on-comment.yml new file mode 100644 index 0000000000..0c614e126f --- /dev/null +++ b/.github/workflows/trigger-on-comment.yml @@ -0,0 +1,27 @@ +name: Trigger on Comment + +on: + issue_comment: + types: [created, edited] + +jobs: + trigger-workflows: + if: github.event.issue.pull_request && contains(github.event.comment.body, '/oktotest') + runs-on: ubuntu-latest + steps: + - name: Check if commenter is authorized + id: check-authorization + run: | + COMMENTER=$(jq --raw-output .comment.user.login "$GITHUB_EVENT_PATH") + APPROVERS=$(yq e '.approvers[]' OWNERS) + if echo "$APPROVERS" | grep -Fxq "$COMMENTER"; then + echo "::set-output name=authorized::true" + else + echo "::set-output name=authorized::false" + fi + - name: Dispatch event + if: steps.check-authorization.outputs.authorized == 'true' + uses: peter-evans/repository-dispatch@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + event-type: ok-to-test diff --git a/ops/terraform/dev.tfvars b/ops/terraform/dev.tfvars index 99532f3da0..6fa6ff1d00 100644 --- a/ops/terraform/dev.tfvars +++ b/ops/terraform/dev.tfvars @@ -27,3 +27,4 @@ num_gpu_machines = 0 log_level = "debug" otel_collector_version = "0.70.0" otel_collector_endpoint = "http://localhost:4318" +web_ui_enabled = true diff --git a/ops/terraform/main.tf b/ops/terraform/main.tf index d04d4b188b..ec6c73d682 100644 --- a/ops/terraform/main.tf +++ b/ops/terraform/main.tf @@ -54,7 +54,7 @@ export BACALHAU_BRANCH="${var.bacalhau_branch}" export BACALHAU_PORT="${var.bacalhau_port}" export BACALHAU_UNSAFE_CLUSTER="${var.bacalhau_unsafe_cluster ? "yes" : ""}" export BACALHAU_NODE_TYPE="${count.index == 0 ? "requester,compute" : "compute"}" -export BACALHAU_NODE_WEBUI="${count.index == 0 ? "true" : "false"}" +export BACALHAU_NODE_WEBUI="${var.web_ui_enabled && count.index == 0 ? "true" : "false"}" export BACALHAU_NODE0_UNSAFE_ID="QmUqesBmpC7pSzqH86ZmZghtWkLwL6RRop3M1SrNbQN5QD" export GPU_NODE="${count.index >= var.instance_count - var.num_gpu_machines ? "true" : "false"}" export GRAFANA_CLOUD_PROMETHEUS_USER="${var.grafana_cloud_prometheus_user}" diff --git a/ops/terraform/prod.tfvars b/ops/terraform/prod.tfvars index 72ccaa2f76..5d3cbef0ba 100644 --- a/ops/terraform/prod.tfvars +++ b/ops/terraform/prod.tfvars @@ -1,4 +1,4 @@ -bacalhau_version = "v1.3.2" +bacalhau_version = "v1.4.0" bacalhau_port = "1235" bacalhau_environment = "production" ipfs_version = "v0.12.2" @@ -31,3 +31,4 @@ public_ip_addresses = ["35.245.161.250", "34.86.254.26", "35.245.215.155 log_level = "debug" otel_collector_version = "0.70.0" otel_collector_endpoint = "http://localhost:4318" +web_ui_enabled = false \ No newline at end of file diff --git a/ops/terraform/stage.tfvars b/ops/terraform/stage.tfvars index 4106661612..f220f3a4cb 100644 --- a/ops/terraform/stage.tfvars +++ b/ops/terraform/stage.tfvars @@ -1,4 +1,4 @@ -bacalhau_version = "v1.3.2" +bacalhau_version = "v1.4.0-rc9" bacalhau_branch = "" # deploy from a branch instead of the version above bacalhau_port = "1235" bacalhau_environment = "staging" @@ -28,3 +28,4 @@ public_ip_addresses = ["34.85.228.65", "34.86.73.105", "34.150.138.100", log_level = "debug" otel_collector_version = "0.70.0" otel_collector_endpoint = "http://localhost:4318" +web_ui_enabled = true \ No newline at end of file diff --git a/ops/terraform/variables.tf b/ops/terraform/variables.tf index 3bba0a20d9..ffcf98551a 100644 --- a/ops/terraform/variables.tf +++ b/ops/terraform/variables.tf @@ -207,4 +207,9 @@ variable "docker_password" { type = string default = "" sensitive = true +} + +variable "web_ui_enabled" { + description = "Enable or disable the web UI" + type = bool } \ No newline at end of file diff --git a/pkg/models/plan.go b/pkg/models/plan.go index 2b49628cce..193e08314a 100644 --- a/pkg/models/plan.go +++ b/pkg/models/plan.go @@ -82,8 +82,8 @@ func (p *Plan) MarkJobRunningIfEligible() { return } - // Only proceed if the current job state is "Pending". - if p.Job.State.StateType != JobStateTypePending { + // Only proceed if the current job state is "Pending" or "Queued". + if p.Job.State.StateType != JobStateTypePending && p.Job.State.StateType != JobStateTypeQueued { return }