Continuous benchmarking - Image size experiments #175
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous benchmarking - Image size experiments | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 12 * * 1" | |
| jobs: | |
| build_client: | |
| name: Build framework | |
| runs-on: ubuntu-latest | |
| env: | |
| working-directory: ./src | |
| steps: | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Go 1.21 | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.21 | |
| - name: Build client binary | |
| working-directory: ${{ env.working-directory }} | |
| run: env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o stellar main.go | |
| - name: Package client artifact | |
| working-directory: ${{ env.working-directory }} | |
| run: tar -czvf build.tar ./stellar | |
| - name: Upload client artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| path: ${{ env.working-directory }}/build.tar | |
| retention-days: 1 | |
| cold-image-size-50-aws: | |
| name: AWS 50MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, aws ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Configure AWS credentials using EASE lab account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} | |
| aws-region: us-west-1 | |
| - name: Set up Node.js 16.16.0 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: AWS Image Size 50MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-aws.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-50-aws | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-image-size-100-aws: | |
| name: AWS 100MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, aws ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Configure AWS credentials using EASE lab account | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} | |
| aws-region: us-west-1 | |
| - name: Set up Node.js 16.16.0 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: AWS Image Size 100MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -a 356764711652 -o latency-samples -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-aws.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-100-aws | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-aws","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-image-size-50-azure: | |
| name: Azure 50MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, azure ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node 16.16.0 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Azure Image Size 50MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-azure.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-50-azure | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-image-size-100-azure: | |
| name: Azure 100MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, azure ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - name: Set up Node 16.16.0 | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: Azure Image Size 100MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-azure.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-100-azure | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-azure","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-image-size-50-gcr: | |
| name: GCR 50MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, gcr ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - id: auth | |
| name: Configure GCR credentials | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCR_CREDENTIALS }} | |
| - name: Set up gcloud | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| version: ">= 363.0.0" | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: GCR Image Size 50MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-50-gcr.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-50-gcr | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-50-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| cold-image-size-100-gcr: | |
| name: GCR 100MB image size experiment | |
| needs: build_client | |
| runs-on: [ self-hosted, gcr ] | |
| timeout-minutes: 1200 | |
| env: | |
| working-directory: src | |
| DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} | |
| steps: | |
| - name: "Cleanup build folder" | |
| run: | | |
| sudo rm -rf ./* || true | |
| sudo rm -rf ./.??* || true | |
| - name: Check out code into the Go module directory | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: continuous-benchmarking | |
| - id: auth | |
| name: Configure GCR credentials | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCR_CREDENTIALS }} | |
| - name: Set up gcloud | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| version: ">= 363.0.0" | |
| - name: Download client artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: STeLLAR-build | |
| - name: Untar client build | |
| working-directory: ${{ env.working-directory }} | |
| run: tar --strip-components=1 -xvf ../build.tar -C . | |
| - name: GCR Image Size 100MB | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1440 | |
| max_attempts: 5 | |
| retry_wait_seconds: 60 | |
| command: cd src && ./stellar -c ../continuous-benchmarking/experiments/cold-function-invocations/image-size/cold-image-size-100-gcr.json -db | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: cold-image-size-100-gcr | |
| path: ${{ env.working-directory }}/latency-samples | |
| - name: Add zero'ed data (Failed experiment) | |
| if: ${{ failure() }} | |
| env: | |
| DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}} | |
| run: | | |
| curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"cold-image-size-100-gcr","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL | |
| - name: Send Slack message using Incoming Webhooks | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Scheduled experiment (Job status)" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow Name*: ${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Name*: ${{ github.job }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Job Status*: ${{ job.status }} ❌" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "<!channel>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |