Merge branch 'main' into dependabot/pip/pdoc-14.4.0

.github/workflows/annotated-pkgs.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         env:
           FORCE_COLOR: 0
         with:

.github/workflows/cypress-end-to-end-tests.yml

@@ -34,11 +34,11 @@ jobs:
       # Workaround to avoid Post Use step failures. See: https://github.com/actions/setup-node/issues/317
       - run: mkdir -p /home/runner/work/Mephisto/Mephisto/.yarn/cache
         continue-on-error: true
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           cache: "yarn"
           node-version: 16
-      - uses: dorny/paths-filter@v2
+      - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -77,12 +77,12 @@ jobs:
         uses: actions/checkout@v2
 
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 
@@ -124,12 +124,12 @@ jobs:
         uses: actions/checkout@v2
 
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 
@@ -171,12 +171,12 @@ jobs:
         uses: actions/checkout@v2
 
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 
@@ -217,12 +217,12 @@ jobs:
         uses: actions/checkout@v2
 
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 
@@ -265,12 +265,12 @@ jobs:
         uses: actions/checkout@v2
 
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 
@@ -313,12 +313,12 @@ jobs:
       - name: 🔀 Checking out repo
         uses: actions/checkout@v2
       - name: 🐍 Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
       - name: 🪨 Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
 

.github/workflows/deploy-docs.yml

@@ -16,13 +16,13 @@ jobs:
         working-directory: docs/web
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         env:
           FORCE_COLOR: 0
         with:
           node-version: 16.16.0
           cache: yarn
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: 🤖 Install Mephisto

.github/workflows/npm-check.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Install Mephisto

.github/workflows/pre-commit.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - uses: pre-commit/[email protected]

.github/workflows/python-test.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -27,7 +27,7 @@ jobs:
         run: |
           python -m pytest -v -m "not req_creds and not prolific" --cov --cov-report=xml
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.xml
           fail_ci_if_error: true

.github/workflows/test-deploy-docs.yml

@@ -18,11 +18,11 @@ jobs:
         working-directory: docs/web
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v4
         with:
           node-version: 16.16.0
           cache: yarn
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
 

.github/workflows/version-sync.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Install Mephisto

CONTRIBUTING.md

@@ -25,7 +25,7 @@ We actively welcome your pull requests.
 ## Cypress Testing
 For cypress testing the base url is: http://localhost:3000/?worker_id=x&assignment_id=1
 
-### Running end to end tests on a task:
+### Running end-to-end tests on a task:
 1. Run the task by running python run_task.py in the appropriate task folder
 2. In a separate terminal window go into the webapp directory and run `npm run test`
 3. This should open a cypress app

Dockerfile

@@ -1,14 +1,13 @@
 # Using the -slim version below for minimal size. You may want to
 # remove -slim, or switch to -alpine if encountering issues
-ARG BASE_TAG=python3.9-nodejs15-slim
+ARG BASE_TAG=python3.9-nodejs16-slim
 ARG BASE_IMAGE=nikolaik/python-nodejs:$BASE_TAG
 
 FROM $BASE_IMAGE
 
 # ec2 architect requires `ssh-keygen` util, so we need to install it.
 # Firstly, remove `yarn` repo as it causes error that stops building a container. Error:
 # (Error: The repository 'https://dl.yarnpkg.com/debian stable InRelease' is not signed)
-RUN rm /etc/apt/sources.list.d/yarn.list
 RUN apt update
 RUN apt install keychain -y
 

README.md

@@ -45,7 +45,7 @@ If you use Mephisto in your work, please consider citing the project. It helps u
   keywords = {Artificial Intelligence (cs.AI), Human-Computer Interaction (cs.HC), FOS: Computer and information sciences, FOS: Computer and information sciences},
   title = {Mephisto: A Framework for Portable, Reproducible, and Iterative Crowdsourcing},
   publisher = {arXiv},
-  year = {2023},
+  year = {2024},
   copyright = {arXiv.org perpetual, non-exclusive license}
 }
 ```

docker/docker-compose.dev.vscode.yml

@@ -20,7 +20,7 @@ services:
       - ./aws_credentials:/root/.aws/credentials
       - ./ssh_known_hosts:/root/.ssh/known_hosts
     entrypoint: /entrypoint.sh
-    env_file: envs/env.local
+    env_file: envs/env.dev
     command: [
         "sh",
         "-c",

docker/envs/env.dev

@@ -1,13 +1,15 @@
-PROLIFIC_API_KEY="..."
-
-MTURK_SANDBOX_NAME="..."
-MTURK_SANDBOX_ACCESS_KEY_ID="..."
-MTURK_SANDBOX_SECRET_ACCESS_KEY="..."
-
-AWS_ACCESS_KEY_ID=...
-AWS_SECRET_ACCESS_KEY=...
-AWS_DEFAULT_REGION=...
-
+# Paste Prolific credentials here if you're using Prolific provider
+PROLIFIC_API_KEY=your_key
+
+# Uncomment and paste Mturk credentials here if you're using Mturk sandbox provider
+# MTURK_SANDBOX_NAME=your_name
+# MTURK_SANDBOX_ACCESS_KEY_ID=your_id
+# MTURK_SANDBOX_SECRET_ACCESS_KEY=your_key
+
+# Paste AWS credentials here if you're using presigned S3 URLs
+AWS_ACCESS_KEY_ID=your_id
+AWS_SECRET_ACCESS_KEY=your_key
+AWS_DEFAULT_REGION=us-east-1
 S3_URL_EXPIRATION_MINUTES=60
 
 CYPRESS_CACHE_FOLDER=/tmp

docs/web/docs/guides/upgrade_to_1/_category_.yml → docs/legacy/upgrade_to_v1/_category_.yml

@@ -2,6 +2,6 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-label: "Upgrading to Version 1.0"
+label: "Upgrading Mephisto"
 collapsed: true
 position: 5

docs/web/docs/guides/upgrade_to_1/guide.md → docs/legacy/upgrade_to_v1/guide.md

@@ -1,14 +1,13 @@
-<!---
-  Copyright (c) Meta Platforms and its affiliates.
-  This source code is licensed under the MIT license found in the
-  LICENSE file in the root directory of this source tree.
--->
-
 ---
+
+# Copyright (c) Meta Platforms and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
 sidebar_position: 1
 ---
 
-# Upgrade Guide
+# Upgrade to v1.0
 
 1. Update the Mephisto library to v1.
     - If you set up Mephisto using `pip install -e`, ensure you pull the latest version down from the git repo.
@@ -18,7 +17,7 @@ sidebar_position: 1
     npm install --save mephisto-task@2
     npm install --save bootstrap-chat@2 # if applicable
     ```
-3. For webapps using `mephisto-task`: Migrate any usage of `sendMessage` in your front-end code to `sendLiveUpdate`.   
+3. For webapps using `mephisto-task`: Migrate any usage of `sendMessage` in your front-end code to `sendLiveUpdate`.
 4. [Migrate your run scripts](../run_scripts) to use the newly introduced syntax, aimed to reduce boilerplate.
 
 

docs/web/docs/guides/upgrade_to_1/run_scripts.md → docs/legacy/upgrade_to_v1/run_scripts.md

@@ -1,14 +1,13 @@
-<!---
-  Copyright (c) Meta Platforms and its affiliates.
-  This source code is licensed under the MIT license found in the
-  LICENSE file in the root directory of this source tree.
--->
-
 ---
+
+# Copyright (c) Meta Platforms and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
 sidebar_position: 2
 ---
 
-# Migrating Run Scripts
+# Run migrations scripts for v1.0
 
 Prior to Mephisto 1.0, our run scripts relied directly on Hydra semantics to settle in to launch things. This led to some unclear import ordering (not including `"_self_"` at times), boilerplate on registering the configs, and other issues.
 

docs/storybook/package.json

@@ -40,10 +40,10 @@
     "@storybook/addon-actions": "^6.5.10",
     "@storybook/addon-docs": "^6.5.10",
     "@storybook/addon-essentials": "^6.5.10",
-    "@storybook/addon-links": "^6.5.10",
+    "@storybook/addon-links": "^8.0.10",
     "@storybook/node-logger": "^6.5.10",
     "@storybook/preset-create-react-app": "^4.1.2",
-    "@storybook/react": "^6.5.10",
+    "@storybook/react": "^8.0.10",
     "prop-types": "^15.8.1",
     "webpack-hot-middleware": "^2.25.2"
   },

docs/web/README.md

@@ -20,6 +20,12 @@ $ yarn
 $ yarn start
 ```
 
+or inside Docker
+
+```
+$ yarn start:docker
+```
+
 This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
 
 ### Build

docs/web/blog/2022-03-22-mephisto-1.0.md

@@ -48,4 +48,4 @@ We've migrated to an asyncio-based design under-the-hood to prevent threading an
 **Improved Developer Tooling**: For Mephisto developers and people interested in working on Mephisto, we've built out new tooling to ensure that our releases are automated/tested, scripts to facilitate entering dev mode, and documentation specifically for you. We've also refactored parts of our underlying codebase to make it easier to jump into and work with.
 
 
-To upgrade your existing projects to Mephisto 1.0, [see our upgrade guide here](/docs/guides/upgrade_to_1/guide).
+To upgrade your existing projects to Mephisto 1.0, see our upgrade guide.

docs/web/docs/explanations/architect_api.md

@@ -1,8 +1,11 @@
-<!---
-  Copyright (c) Meta Platforms and its affiliates.
-  This source code is licensed under the MIT license found in the
-  LICENSE file in the root directory of this source tree.
--->
+---
+
+# Copyright (c) Meta Platforms and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+sidebar_position: 1
+---
 
 # Architects, Routers, and `mephisto-task`: The Architect API.
 

docs/web/docs/explanations/architecture_overview.md

@@ -1,8 +1,11 @@
-<!---
-  Copyright (c) Meta Platforms and its affiliates.
-  This source code is licensed under the MIT license found in the
-  LICENSE file in the root directory of this source tree.
--->
+---
+
+# Copyright (c) Meta Platforms and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+sidebar_position: 2
+---
 
 # Mephisto Architecture
 
@@ -141,4 +144,4 @@ The core responsibility of the Operator is to, given a task to deploy and config
 ______
 
 ### DataBrowser
-As the Mephisto data model is all stored to disk, we also provide a `MephistoDataBrowser` class that has common methods used in other parts of the workflow, such as reviewing work and analyzing trends.
+As the Mephisto data model is all stored to disk, we also provide a `MephistoDataBrowser` class that has common methods used in other parts of the workflow, such as reviewing work and analyzing trends.