🔀 Merged with main

facebookresearch · Aug 3, 2022 · d65b46c · d65b46c
2 parents 40a09f9 + a5b1f78
commit d65b46c
Show file tree

Hide file tree

Showing 16 changed files with 204 additions and 22 deletions.
diff --git a/.github/workflows/cypress-end-to-end-tests.yml b/.github/workflows/cypress-end-to-end-tests.yml
@@ -10,6 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     # Set job outputs to values from filters step below
     outputs:
+      simple_static_task: ${{ steps.filter.outputs.simple_static_task }}
       static_react_task: ${{ steps.filter.outputs.static_react_task }}
       static_react_task_with_tips: ${{ steps.filter.outputs.static_react_task_with_tips }}
       mnist: ${{ steps.filter.outputs.mnist }}
@@ -29,6 +30,8 @@ jobs:
         id: filter
         with:
           filters: |
+            simple_static_task:
+              - 'examples/simple_static_task/**'
             static_react_task:
               - 'examples/static_react_task/**'
             static_react_task_with_tips:
@@ -52,6 +55,51 @@ jobs:
             mephisto-worker-addons:
               - 'packages/mephisto-worker-addons/src/**'
 
+  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881
+  simple_static_task:
+    needs: changes
+    if: ${{ (needs.changes.outputs.simple_static_task == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🔀 Checking out repo
+        uses: actions/checkout@v2
+
+      - name: 🐍 Installing python
+        uses: actions/setup-python@v2
+
+      - name: 🪨 Setup Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+
+      - name: 🤖 Install Mephisto
+        run: pip install -e .
+
+      - name: 🖋 Create data directory
+        run: mkdir -p ~/mephisto/data
+
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📦 Setting up mephisto-task package
+        run: |
+          cd packages/mephisto-task
+          yarn install
+          yarn build
+          npm link
+
+      - name: ⌛️ Running cypress tests
+        uses: cypress-io/[email protected]
+        with:
+          install: false
+          browser: chrome
+          project: ./mephisto/abstractions/blueprints/static_html_task/source
+          config-file: ./cypress.config.js
+          start: python ./examples/simple_static_task/static_test_script.py
+          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
+          command-prefix: yarn dlx
+          headless: true
+
   # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795
   static-react-task:
     needs: changes

diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
@@ -20,8 +20,20 @@ jobs:
         with:
           node-version: 16.x
           cache: yarn
+      - uses: actions/setup-python@v2
 
-      - name: Auto Generate Blueprint Docs
+      - name: 🤖 Install Mephisto
+        run: |
+          cd ../../
+          pip install -e .
+
+      - name: 🖋 Create data directory
+        run: mkdir -p ~/mephisto/data
+
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📝 Auto Generate Blueprint Docs
         run: |
           cd ../../mephisto/scripts/local_db/gh_actions
           python auto_generate_blueprint.py

diff --git a/.github/workflows/test-deploy-docs.yml b/.github/workflows/test-deploy-docs.yml
@@ -22,6 +22,26 @@ jobs:
         with:
           node-version: 16.x
           cache: yarn
+      - uses: actions/setup-python@v2
+
+      - name: 🤖 Install Mephisto
+        run: |
+          cd ../../
+          pip install -e .
+
+      - name: 🖋 Create data directory
+        run: mkdir -p ~/mephisto/data
+
+      - name: 📂 Set the data directory
+        run: mephisto config core.main_data_directory ~/mephisto/data
+
+      - name: 📝 Auto Generate Blueprint Docs
+        run: |
+          cd ../../mephisto/scripts/local_db/gh_actions
+          python auto_generate_blueprint.py
+          python auto_generate_architect.py
+          python auto_generate_requester.py
+          python auto_generate_provider.py
 
       - name: Install dependencies
         run: yarn install --frozen-lockfile

diff --git a/docs/web/README.md b/docs/web/README.md
@@ -1,4 +1,4 @@
-# Website
+# Mephisto Docs Website
 
 This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator.
 

diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.md b/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.md
@@ -33,7 +33,7 @@ def validate_screening_unit(unit: "Unit"):
 shared_state = SharedTaskState(
     ...
     screening_data_factory=False
-    on_unit_submitted=ScreenTaskRequired.create_validation_function(cfg.mephisto, validate_gold_unit)
+    on_unit_submitted=ScreenTaskRequired.create_validation_function(cfg.mephisto, validate_screening_unit)
 )
 shared_state.qualifications += ScreenTaskRequired.get_mixin_qualifications(cfg.mephisto, shared_state)
 ...

diff --git a/examples/simple_static_task/server_files/demo_task.html b/examples/simple_static_task/server_files/demo_task.html
@@ -31,7 +31,12 @@
       <div class="modal-body" id="modal">&nbsp;</div>
 
       <div class="modal-footer">
-        <button class="btn btn-secondary" data-dismiss="modal" type="button">
+        <button
+          data-cy="close-modal-button"
+          class="btn btn-secondary"
+          data-dismiss="modal"
+          type="button"
+        >
           Close
         </button>
       </div>
@@ -54,11 +59,15 @@
   <div class="row col-xs-12 col-md-12">
     <!-- Instructions -->
     <div class="panel panel-primary">
-      <div class="panel-heading">
+      <div class="panel-heading" data-cy="instructions-panel-header">
         <strong> Instructions </strong>
       </div>
 
-      <div class="panel-body" id="instructions">
+      <div
+        class="panel-body"
+        id="instructions"
+        data-cy="instructions-panel-body"
+      >
         <p>
           Below you are given a character name and description for someone from
           a fantasy story. Please rate how well the character description
@@ -92,11 +101,11 @@
     <!-- Content Body -->
 
     <div class="input_location">
-      <p>
+      <p data-cy="character-name-paragraph">
         <strong> Character name: </strong>
         <span id="char"> ${character_name} </span>
       </p>
-      <p>
+      <p data-cy="character-description-paragraph">
         <strong> Description: </strong>
         <span id="persona"> ${character_description} </span>
       </p>
@@ -105,14 +114,26 @@
     <section>
       <fieldset>
         <div class="input-group">
-          <label>Rate the matching of character name to description</label>
-          <select class="form-control" name="rating">
+          <label for="character-dropdown"
+            >Rate the matching of character name to description</label
+          >
+          <select
+            id="character-dropdown"
+            class="form-control"
+            name="rating"
+            data-cy="character-dropdown"
+          >
             <option selected="selected" value="">(select one)</option>
             <option value="good">Good</option>
             <option value="passable">Passable</option>
             <option value="bad">Bad</option>
           </select>
-          <input type="file" name="file1" id="upload_files_1" />
+          <input
+            data-cy="character-file-input"
+            type="file"
+            name="file1"
+            id="upload_files_1"
+          />
         </div>
       </fieldset>
     </section>

diff --git a/mephisto/abstractions/blueprints/parlai_chat/parlai_chat_agent_state.py b/mephisto/abstractions/blueprints/parlai_chat/parlai_chat_agent_state.py
@@ -69,7 +69,6 @@ def _load_data(self) -> None:
 
     def get_data(self) -> Dict[str, Any]:
         """Return dict with the messages of this agent"""
-        print(self.metadata)
         return {
             "outputs": {
                 "messages": self.messages,

diff --git a/mephisto/abstractions/blueprints/static_html_task/source/cypress.config.js b/mephisto/abstractions/blueprints/static_html_task/source/cypress.config.js
@@ -0,0 +1,7 @@
+module.exports = {
+  video: false,
+
+  e2e: {
+    baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1",
+  },
+};
diff --git a/...isto/abstractions/blueprints/static_html_task/source/cypress/e2e/simple_static_task.cy.js b/...isto/abstractions/blueprints/static_html_task/source/cypress/e2e/simple_static_task.cy.js
@@ -0,0 +1,53 @@
+describe("Loads simple_static_task", () => {
+  it("Makes request for agent", () => {
+    cy.intercept({ pathname: "/request_agent" }).as("agentRequest");
+    cy.visit("/");
+    cy.wait("@agentRequest").then((interception) => {
+      expect(interception.response.statusCode).to.eq(200);
+    });
+  });
+  it("Loads correct elements", () => {
+    cy.get('[data-cy="close-modal-button"]');
+    cy.get('[data-cy="instructions-panel-header"]');
+    cy.get('[data-cy="instructions-panel-body"]');
+    cy.get('[data-cy="character-name-paragraph"]');
+    cy.get('[data-cy="character-description-paragraph"]');
+    cy.get('[data-cy="character-dropdown"]');
+    cy.get('[data-cy="submit-button"]');
+  });
+});
+
+describe("Submits the html_static_task", () => {
+  it("Closing starting modal", () => {
+    cy.get('[data-cy="close-modal-button"]').as("modalButton").click();
+    cy.get("@modalButton").should("not.be.visible");
+  });
+  it("Select a character name description", () => {
+    cy.get('[data-cy="character-dropdown"]').select("Good");
+    cy.get('[data-cy="character-dropdown"]').select("Bad");
+  });
+  it("Upload a file", () => {
+    cy.fixture("bliss.png").then((fileContent) => {
+      cy.get('[data-cy="character-file-input"]').attachFile({
+        fileContent: fileContent.toString(),
+        fileName: "bliss.png",
+        mimeType: "image/png",
+      });
+    });
+  });
+  it("Submit the task", () => {
+    cy.on("window:alert", (txt) => {
+      expect(txt).to.contains(
+        'The task has been submitted! Data: {"rating":"bad"'
+      );
+      expect(txt).to.contains('"name":"bliss.png"');
+      expect(txt).to.contains('"size":51824');
+      expect(txt).to.contains('"type":"image/png"');
+    });
+    cy.intercept({ pathname: "/submit_task" }).as("submitTask");
+    cy.get('[data-cy="submit-button"]').click();
+    cy.wait("@submitTask").then((interception) => {
+      expect(interception.response.statusCode).to.eq(200);
+    });
+  });
+});
diff --git a/...isto/abstractions/blueprints/static_html_task/source/cypress/fixtures/bliss.png b/...isto/abstractions/blueprints/static_html_task/source/cypress/fixtures/bliss.png
diff --git a/mephisto/abstractions/blueprints/static_html_task/source/cypress/support/commands.js b/mephisto/abstractions/blueprints/static_html_task/source/cypress/support/commands.js
@@ -0,0 +1 @@
+import "cypress-file-upload";
diff --git a/mephisto/abstractions/blueprints/static_html_task/source/cypress/support/e2e.js b/mephisto/abstractions/blueprints/static_html_task/source/cypress/support/e2e.js
@@ -0,0 +1 @@
+import "./commands";
diff --git a/mephisto/abstractions/blueprints/static_html_task/source/dev/app.jsx b/mephisto/abstractions/blueprints/static_html_task/source/dev/app.jsx
@@ -51,6 +51,18 @@ function MainApp() {
       handleSubmit(objData);
     } else {
       formData.append("USED_AGENT_ID", agentId);
+
+      objData.file1.size === 0
+        ? (objData.file1 = {})
+        : (objData.file1 = {
+            lastModified: objData.file1.lastModified
+              ? objData.file1.lastModified
+              : -1,
+            name: objData.file1.name ? objData.file1.name : "",
+            size: objData.file1.size ? objData.file1.size : -1,
+            type: objData.file1.type ? objData.file1.type : "",
+          });
+
       formData.append("final_string_data", JSON.stringify(objData));
       postData("/submit_task", formData)
         .then((data) => {
@@ -135,6 +147,7 @@ function SubmitFrame({ children, onSubmit, currentTask }) {
                 id="html-task-submit-button"
                 type="submit"
                 disabled={submitting}
+                data-cy="submit-button"
               >
                 <span
                   style={{ marginRight: 5 }}

diff --git a/mephisto/abstractions/blueprints/static_html_task/source/package.json b/mephisto/abstractions/blueprints/static_html_task/source/package.json
@@ -4,7 +4,8 @@
   "description": "",
   "main": "webpack.config.js",
   "scripts": {
-    "dev": "webpack --mode development"
+    "dev": "webpack --mode development",
+    "test": "cypress open"
   },
   "keywords": [],
   "author": "",
@@ -26,6 +27,8 @@
     "@babel/preset-react": "^7.0.0",
     "babel-loader": "^8.0.2",
     "css-loader": "^6.7.1",
+    "cypress": "^10.4.0",
+    "cypress-file-upload": "^5.0.8",
     "file-loader": "^6.0.0",
     "style-loader": "^1.3.0",
     "url-loader": "^4.1.1",

diff --git a/mephisto/operations/worker_pool.py b/mephisto/operations/worker_pool.py
@@ -265,26 +265,30 @@ async def _assign_unit_to_agent(
                     agent,
                 )
             else:
-                # See if the concurrent unit is ready to launch
                 assignment = await loop.run_in_executor(None, unit.get_assignment)
+
+                # Set status to waiting
+                agent.update_status(AgentState.STATUS_WAITING)
+
+                # See if the concurrent assignment is ready to launch
                 logger.debug(f"Attempting to launch {assignment}.")
                 agents = await loop.run_in_executor(None, assignment.get_agents)
                 if None in agents:
-                    agent.update_status(AgentState.STATUS_WAITING)
                     return  # need to wait for all agents to be here to launch
 
+                for queried_agent in agents:
+                    if queried_agent.get_status() != AgentState.STATUS_WAITING:
+                        logger.debug(f"Delaying launch of {assignment}, should retry.")
+                        return  # Need to wait for all agents to be waiting to launch
+
+                # Mypy not-null cast
                 non_null_agents = [a for a in agents if a is not None]
                 # Launch the backend for this assignment
                 registered_agents = [
-                    self.agents.get(a.get_agent_id())
+                    self.agents[a.get_agent_id()]
                     for a in non_null_agents
                     if a is not None
                 ]
-                if None in registered_agents:
-                    # an Agent has been matched to the Assignment, but isn't yet watched
-                    # by the pool. Wait for that thread to catch up
-                    logger.debug(f"Delaying launch of {assignment}, should retry.")
-                    return
 
                 live_run.task_runner.execute_assignment(assignment, registered_agents)
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -47,7 +47,7 @@ exclude = [
 python = "^3.7"
 requests = "^2.22"
 sh = "^1.12"
-hydra-core = "^1.1.0"
+hydra-core = "^1.2.0"
 tqdm = "^4.50.2"
 websockets = "^10.1"
 pyyaml = "^5.4.0"