From 221bae32a6670d1f3cbfe1500c97e287c56e152e Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Fri, 27 Oct 2023 11:15:45 -0400 Subject: [PATCH 1/2] Added step-by-step Mephisto guide --- README.md | 2 + .../hydra_configs/conf/prolific_example.yaml | 2 +- mephisto/README.md | 252 +++++++++++++++++- 3 files changed, 253 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c1cc348f0..7357eca8d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ You can find complete details about the project on our [docs website](https://me [Get started in 10 minutes](https://mephisto.ai/docs/guides/quickstart) +[How to run projects](mephisto/README.md) + ## Want to help? Check out our [guidelines for contributing](https://github.com/facebookresearch/Mephisto/blob/main/CONTRIBUTING.md) and then take a look at some of our tagged issues: [good first issue](https://github.com/facebookresearch/Mephisto/labels/good%20first%20issue), [help wanted](https://github.com/facebookresearch/Mephisto/labels/help%20wanted). diff --git a/examples/simple_static_task/hydra_configs/conf/prolific_example.yaml b/examples/simple_static_task/hydra_configs/conf/prolific_example.yaml index da5db7243..0b747ab38 100644 --- a/examples/simple_static_task/hydra_configs/conf/prolific_example.yaml +++ b/examples/simple_static_task/hydra_configs/conf/prolific_example.yaml @@ -42,5 +42,5 @@ mephisto: - 6463d44ed1b61a8fb4e0765a - 6463d488c2f2821eaa2fa13f - name: "ApprovalRateEligibilityRequirement" - minimum_approval_rate: 0 + minimum_approval_rate: 1 maximum_approval_rate: 100 diff --git a/mephisto/README.md b/mephisto/README.md index 6734dde4e..811c84712 100644 --- a/mephisto/README.md +++ b/mephisto/README.md @@ -4,12 +4,260 @@ LICENSE file in the root directory of this source tree. --> -# Mephisto +# About Mephisto + +## Purpose + +The purpose of Mephisto is to run a data collection project that you can launch it locally or on a remote machine. A launched project (TaskRun) does the following: + +- Builds necessary infra with an Architect (local-based mock, or cloud-based EC2/Heroku) +- The infra server does the following, until specified number of Task Units is reached: + - It generates a link where a worker can provide their project input (Task Unit) + - The links are sent to a human cloud (Provider), and a worker clicks on the link + - The links displays a UI task template (Task App) to the worker; submitting the task sends worker's input to the infra server + - The infra server sends (via a web socket) data to the TaskRun that you're running locally + - TaskRun stores the data in local database/files, and dispatches new Task Units as needed +- Finally, you can review worker input using Review App, and export the data out of the database as needed. + +--- + +## Architecture + This is the main package directory, containing all of the core workings of Mephisto. They roughly follow the divisions noted in the [architecture overview doc](https://github.com/facebookresearch/Mephisto/blob/main/docs/architecture_overview.md#agent). The breakdown is as following: - `abstractions`: Contains the interface classes for the core abstractions in Mephisto, as well as implementations of those interfaces. These are the Architects, Blueprints, Crowd Providers, and Databases. -- `client`: Contains user interfaces for using Mephisto at a very high level. Primarily comprised of the python code for the cli and the web views. +- `client`: Contains user interfaces for using Mephisto at a very high level. Primarily comprised of the python code for the cli and the web views, such as Review App. - `data_model`: Contains the data model components as described in the architecture document. These are the relevant data structures that build upon the underlying MephistoDB, and are utilized throughout the Mephisto codebase. - `operations`: Contains low-level operational code that performs more complex functionality on top of the Mephisto data model. - `scripts`: Contains commonly executed convenience scripts for Mephisto users. - `tools`: Contains helper methods and modules that allow for lower-level access to the Mephisto data model than the clients provide. Useful for creating custom workflows and scripts that are built on Mephisto. + +#### Providers + +Provider is a host of human cloud, or worker crowd. Currently the following Providers are supported: + +- `Mock` - runs locally, assuming your machine has a static IP. Used for testing and elementary projects. +- `MTurk` - Amazon Mechanical Turk provider (will eventually be deprecated) +- `MTurk Sandbox` - testing setup for Mturk +- `Prolific` - a human cloud prvider that's more reliable than Mturk, albeit with fewer workers + +#### Architects + +Architect is the manager of infrastructure of a project. Currently we support the following infrastructure hosts: + +- `local` - your local machine, assuming it has a static IP address. +- `ec2` - AWS EC2 servers (in FAIR cluster) +- `heroku` - Heroku servers (no future development, in maintenance mode) +- `mock` - used for testing, for example, in Mephisto's unittests + +--- + +# Sample Mephisto projects + +Mephisto repo contains several sample projects. +Let's try to run them with `docker-compose`, starting from the easiest one. +(You can install Docker [here](https://docs.docker.com/engine/install/).) + +If you don't want to engage any cloud infrastructure, you can run all sample projects on a local machine with `mock` architect. Note that for any architect project data will be stored in SQLite databases and local files. + +A few notes: +- If a project breaks and does not shut down cleanly, you may need to remove `tmp` directory in repo root before re-launching. (Otherwise you could see errors like Prometeus cannot start, etc.) +- To see more browser links for task units (assignments) within a TaskRun, check console logs (and remember to use correct port) +- If you terminate a TaskRun, you can launch it again, and results from all TaskRuns will be automatically collated +- Note that most detailed logs are written into files in `outputs/` directory in repo root (not in the console) + +--- + +#### 1. Simple HTML-based task + +A simple project with HTML-based UI task template [simple_static_task](../examples/simple_static_task) + +- Default config file: [example.yaml](../examples/simple_static_task/hydra_configs/conf/example.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + cd /mephisto/examples/simple_static_task && python ./static_test_script.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1) +- Browser page should display an image, instruction, select and file inputs, and a submit button. + +--- + +#### 2. Simple React-based task + +A simple project with React-based UI task template [static_react_task](../examples/static_react_task) + +- Default config file: [example.yaml](../examples/static_react_task/hydra_configs/conf/example.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + cd /mephisto/examples/static_react_task && python ./run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display an instruction line and two buttons (green and red). + +--- + +#### 3. Task with dynamic input + +A more complex example featuring worker-generated dynamic input: [mnist](../examples/remote_procedure/mnist). + +- Default config file: [launch_with_local.yaml](../examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + apt install curl && \ + pip install grafana torch pillow numpy && \ + mephisto metrics install && \ + cd /mephisto/examples/remote_procedure/mnist && python ./run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display instructions and a layout with 3 rectangle fields for drawing numbers with a mouse, each field having inputs at the bottom. + +--- + +## Review collected data + +After running the above examples, your local database will contain some input from workers. You can review it, and assign qualifications to those workers, using Mephisto's Review App. + +- Launch command: +```shell +docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 8081:8000 \ + --rm mephisto_dc \ + mephisto review_app -h 0.0.0.0 -p 8000 -d True -f False -o False +``` +- Browser page: [http://localhost:8081/](http://localhost:8081/). + +The UI is fairly intuitive, and for more details you can consult [README.md for TaskReview app](mephisto/client/review_app/README.md). + +--- + +## Export collected data + +All TaskRun data is stored in `data` directory of repo root: + +- `data/database.db` is Mephisto's main SQLite database with generic objects data + - Note that its DB schema is defined in `mephisto/abstractions/databases/local_database.py` file +- `data/data` folder contains helper files, such as detailed input/output data in JSON +- `data/mock/mock.db` or `data/mturk/mturk.db` or `data/prolific/prolific.db` is Provider-specific SQLite database (DB schema varies greatly depending on the provider). + +Worker responses metadata are in these databases, and actual data of their responses in these folders. After TaskRun is completed and results are reviewed, you can access workers raw responses using `mephisto/tools/examine_utils.py` script + +--- + +# Your Mephisto project + +Here's a list of steps on how to build and run your own custom task. + +## Write Task App code + +In order to launch your own customized project, you will need to write a React app that will display instructions/inputs to workers. You can start by duplicating an existing Task App code (e.g. `examples/static_react_task` directory) and customizing it to your needs. The process goes like this: + +1. Copy `static_react_task` directory to your project directory within Mephisto repo +2. Customize task's back-end code in `run_task.py` script to pass relevant data to `SharedStaticTaskState`, set `shared_state.prolific_specific_qualifications`, `shared_state.qualifications` (for custom qualifications), etc +3. Customize task-related parameters variables in your `conf/.yaml` file as needed. + - Some examples of variables from `blueprint` category are: + - `extra_source_dir`: optional path to sources that Task App may refer to (images, video, css, scripts, etc) + - `data_json`: path to a json file containing task data + - To see other configurable blueprint variables, type `mephisto wut blueprint=static_task` +4. Customize task's front-end code, with starting point being `//webapp/src/components/core_components.jsx` (you caninclude an onboarding step if you like). +5. Add the ability to review results of your task app. In short, you need to implement additional component or logic to render json data that TaskReview app will provide. For more details, read this [doc](mephisto/client/review_app/README.md). +6. Run `run_task.py` to dry-run your task on localhost. +7. Repeat 5 & 6 until you're happy with your task. +8. Launch a small batch with a chosen crowd provider to see how real workers handle your task. +9. Iterate more. +10. Collect some good data. + +--- + +## Configure Task parameters + +This is a sample YAML configuration to run your Task on **AWS EC2** architect with **Prolific** provider + +1. Set Prolific as your provider + ```yaml + defaults: + - /mephisto/provider: prolific + ``` + +2. Set EC2 as an architect + ```yaml + defaults: + - /mephisto/architect: ec2 + mephisto: + architect: + _architect_type: ec2 + profile_name: mephisto-router-iam + subdomain: "2023-08-23.1" + ``` + + Where: + - `profile_name` - EC2 service profile name (used for authentication and domain name selection) + - `subdomain` - must be unique across all TaskRuns. Subdomain on which workers can access their Task Unit + +3. Set Prolific-specific task parameters. Sample parameters could look similar to this: + ```yaml + mephisto: + provider: + prolific_id_option: "url_parameters" + prolific_workspace_name: "My Workspace" + prolific_project_name: "My Project" + prolific_allow_list_group_name: "Allow list" + prolific_block_list_group_name: "Block list" + prolific_eligibility_requirements: + - name: "CustomWhitelistEligibilityRequirement" + white_list: + - 6463d32f50a18041930b71be + - 6463d3922d7d99360896228f + - 6463d40e8d5d2f0cce2b3b23 + - name: "ApprovalRateEligibilityRequirement" + minimum_approval_rate: 1 + maximum_approval_rate: 100 + ``` + + For all available Prolific-specific parameters see `mephisto.abstractions.providers.prolific.prolific_provider.ProlificProviderArgs` class + and [Prolific API Docs](https://docs.prolific.com/docs/api-docs/public/#tag/Studies). + + Note that `prolific_eligibility_requirements` does not include custom worker qualifications, these are maintained in your local Mephisto database. These can be specified in `run_task.py` script (as example, see `examples/simple_static_task/static_test_prolific_script.py`) + +--- + +## Launch TaskRun + +1. Specify auth credentials for your Prolific account. To do so, you need to run command + ```shell + mephisto register prolific name=prolific api_key=API_KEY_FOR_YOUR_PROLIFIC_ACCOUNT + ``` +or simply embed that command into your docker-compose entrypoint script. + +2. Launch a new TaskRun (instead of `examples/simple_static_task` here specify path to your own Task code; `HYDRA_FULL_ERROR=1` is optional and prints out detailed error info) + + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --rm mephisto_dc \ + rm -rf /mephisto/tmp && \ + cd /mephisto/examples/simple_static_task && \ + HYDRA_FULL_ERROR=1 python ./static_test_prolific_script.py + ``` + + This TaskRun script will spin up an EC2 server, upload your React Task App to it, and create a Study on Prolific. Now all eligible workers will see your Task Units (with links poiting to EC2 server) on Prolific, and can complete it. + +3. Leave the Task running in the console until all worker submissions are received. If TaskRun was interrupted, you can restart it using the same commands. After all submissions are received, the Architect will automatically shut down actiive TaskRun. + +--- + +## Process results + +Final steps of reviewing worker submissions and exporting the results will be same as described under sample Mephisto project runs. From 118ff5dba7ffb9dfbfed2744d6fdb1c8282c2c9f Mon Sep 17 00:00:00 2001 From: Seg-mel Date: Thu, 2 Nov 2023 18:11:44 +0300 Subject: [PATCH 2/2] [make-task-review-server] Added creating new qualification Removed redundant modules --- mephisto/client/api.py | 334 ------------------ .../client/src/components/Errors/Errors.css | 1 + .../mnist_core_components_copied.js | 196 ---------- .../pages/TaskPage/ModalForm/ModalForm.css | 11 +- .../pages/TaskPage/ModalForm/ModalForm.tsx | 101 +++++- .../client/src/pages/TaskPage/TaskPage.tsx | 4 +- .../client/src/pages/TaskPage/modalData.tsx | 14 +- .../client/src/types/reviewModal.d.ts | 8 +- .../server/api/views/qualifications_view.py | 2 +- 9 files changed, 114 insertions(+), 557 deletions(-) delete mode 100644 mephisto/client/api.py delete mode 100644 mephisto/client/review_app/client/src/components/mnist_core_components_copied.js diff --git a/mephisto/client/api.py b/mephisto/client/api.py deleted file mode 100644 index 6603d42d2..000000000 --- a/mephisto/client/api.py +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from flask import Blueprint, jsonify, request # type: ignore -from flask import current_app as app # type: ignore -from mephisto.abstractions.database import EntryAlreadyExistsException -from mephisto.data_model.constants.assignment_state import AssignmentState -from mephisto.data_model.task_run import TaskRun -from mephisto.data_model.unit import Unit -from mephisto.data_model.assignment import Assignment -from mephisto.operations.hydra_config import parse_arg_dict, get_extra_argument_dicts -from mephisto.operations.registry import ( - get_blueprint_from_type, - get_crowd_provider_from_type, - get_architect_from_type, - get_valid_blueprint_types, - get_valid_provider_types, - get_valid_architect_types, -) -import sys -import traceback -import os - -api = Blueprint("api", __name__) - - -@api.route("/requesters") -def get_available_requesters(): - db = app.extensions["db"] - requesters = db.find_requesters() - dict_requesters = [r.to_dict() for r in requesters] - return jsonify({"requesters": dict_requesters}) - - -@api.route("/task_runs/running") -def get_running_task_runs(): - """Find running tasks by querying for all task runs that aren't completed""" - db = app.extensions["db"] - task_runs = db.find_task_runs(is_completed=False) - dict_tasks = [t.to_dict() for t in task_runs if not t.get_is_completed()] - live_task_count = len([t for t in dict_tasks if not t["sandbox"]]) - return jsonify( - { - "task_runs": dict_tasks, - "task_count": len(dict_tasks), - "live_task_count": live_task_count, - } - ) - - -@api.route("/task_runs/reviewable") -def get_reviewable_task_runs(): - """ - Find reviewable task runs by querying for all reviewable tasks - and getting their runs - """ - db = app.extensions["db"] - units = db.find_units(status=AssignmentState.COMPLETED) - reviewable_count = len(units) - task_run_ids = set([u.get_assignment().get_task_run().db_id for u in units]) - task_runs = [TaskRun.get(db, db_id) for db_id in task_run_ids] - dict_tasks = [t.to_dict() for t in task_runs] - # TODO(OWN) maybe include warning for auto approve date once that's tracked - return jsonify({"task_runs": dict_tasks, "total_reviewable": reviewable_count}) - - -@api.route("/launch/options") -def launch_options(): - blueprint_types = get_valid_blueprint_types() - architect_types = get_valid_architect_types() - provider_types = get_valid_provider_types() - return jsonify( - { - "success": True, - "architect_types": architect_types, - "provider_types": provider_types, - "blueprint_types": [ - {"name": bp, "rank": idx + 1} for (idx, bp) in enumerate(blueprint_types) - ], - } - ) - - -@api.route("/task_runs/launch", methods=["POST"]) -def start_task_run(): - # Blueprint, CrowdProvider, Architect (Local/Heroku), Dict of arguments - - info = request.get_json(force=True) - input_arg_list = [] - for arg_content in info.values(): - input_arg_list.append(arg_content["option_string"]) - input_arg_list.append(arg_content["value"]) - try: - operator = app.extensions["operator"] - operator.parse_and_launch_run(input_arg_list) - # MOCK? What data would we want to return? - # perhaps a link to the task? Will look into soon! - return jsonify({"status": "success", "data": info}) - except Exception as e: - traceback.print_exc(file=sys.stdout) - return jsonify({"success": False, "msg": f"error in launching job: {str(e)}"}) - - -@api.route("/task_runs//units") -def view_unit(task_id): - # TODO - - # MOCK - return jsonify({"id": task_id, "view_path": "https://google.com", "data": {"name": "me"}}) - - -@api.route("/task_runs/options") -def get_basic_task_options(): - params = get_extra_argument_dicts(TaskRun) - return jsonify({"success": True, "options": params}) - - -@api.route("/requester//options") -def requester_details(requester_type): - crowd_provider = get_crowd_provider_from_type(requester_type) - RequesterClass = crowd_provider.RequesterClass - params = get_extra_argument_dicts(RequesterClass) - return jsonify(params) - - -@api.route("/requester//register", methods=["POST"]) -def requester_register(requester_type): - options = request.get_json() - crowd_provider = get_crowd_provider_from_type(requester_type) - RequesterClass = crowd_provider.RequesterClass - - try: - parsed_options = parse_arg_dict(RequesterClass, options) - except Exception as e: - traceback.print_exc(file=sys.stdout) - return jsonify({"success": False, "msg": f"error in parsing arguments: {str(e)}"}) - if "name" not in parsed_options: - return jsonify({"success": False, "msg": "No name was specified for the requester."}) - - db = app.extensions["db"] - requesters = db.find_requesters(requester_name=parsed_options["name"]) - if len(requesters) == 0: - requester = RequesterClass.new(db, parsed_options["name"]) - else: - requester = requesters[0] - try: - print(parsed_options) - requester.register(parsed_options) - return jsonify({"success": True}) - except Exception as e: - return jsonify({"success": False, "msg": str(e)}) - - -@api.route("/data/submitted_data") -def get_submitted_data(): - try: - task_run_ids = request.args.getlist("task_run_id") - task_names = request.args.getlist("task_name") - assignment_ids = request.args.getlist("assignment_id") - unit_ids = request.args.getlist("unit_ids") - statuses = request.args.getlist("status") - - db = app.extensions["db"] - units = [] - assignments = [] - assert len(task_names) == 0, "Searching via task names not yet supported" - - task_runs = [TaskRun.get(db, task_run_id) for task_run_id in task_run_ids] - for task_run in task_runs: - assignments += task_run.get_assignments() - - assignments += [Assignment.get(db, assignment_id) for assignment_id in assignment_ids] - - if len(statuses) == 0: - statuses = [ - AssignmentState.COMPLETED, - AssignmentState.ACCEPTED, - AssignmentState.REJECTED, - ] - - filtered_assignments = [a for a in assignments if a.get_status() in statuses] - - for assignment in assignments: - units += assignment.get_units() - - units += [Unit.get(db, unit_id) for unit_id in unit_ids] - - all_unit_data = [] - for unit in units: - unit_data = { - "assignment_id": unit.assignment_id, - "task_run_id": unit.task_run_id, - "status": unit.db_status, - "unit_id": unit.db_id, - "worker_id": unit.worker_id, - "data": None, - } - agent = unit.get_assigned_agent() - if agent is not None: - unit_data["data"] = agent.state.get_data() - unit_data["worker_id"] = agent.worker_id - all_unit_data.append(unit_data) - - print(all_unit_data) - return jsonify({"success": True, "units": all_unit_data}) - except Exception as e: - import traceback - - traceback.print_exc() - return jsonify({"success": False, "msg": str(e)}) - - -@api.route("//get_balance") -def get_balance(requester_name): - db = app.extensions["db"] - requesters = db.find_requesters(requester_name=requester_name) - - if len(requesters) == 0: - return jsonify( - { - "success": False, - "msg": f"No requester available with name: {requester_name}", - } - ) - - requester = requesters[0] - return jsonify({"balance": requester.get_available_budget()}) - - -@api.route("/requester//launch_options") -def requester_launch_options(requester_type): - db = app.extensions["db"] - requesters = db.find_requesters(requester_name=requester_name) - - if len(requesters) == 0: - return jsonify( - { - "success": False, - "msg": f"No requester available with name: {requester_name}", - } - ) - provider_type = requesters[0].provider_type - CrowdProviderClass = get_crowd_provider_from_type(requester_type) - params = get_extra_argument_dicts(CrowdProviderClass) - return jsonify({"success": True, "options": params}) - - -@api.route("/blueprints") -def get_available_blueprints(): - blueprint_types = get_valid_blueprint_types() - return jsonify({"success": True, "blueprint_types": blueprint_types}) - - -@api.route("/blueprint//options") -def get_blueprint_arguments(blueprint_type): - if blueprint_type == "none": - return jsonify({"success": True, "options": {}}) - BlueprintClass = get_blueprint_from_type(blueprint_type) - params = get_extra_argument_dicts(BlueprintClass) - return jsonify({"success": True, "options": params}) - - -@api.route("/architects") -def get_available_architects(): - architect_types = get_valid_architect_types() - return jsonify({"success": True, "architect_types": architect_types}) - - -@api.route("/architect//options") -def get_architect_arguments(architect_type): - if architect_type == "none": - return jsonify({"success": True, "options": {}}) - ArchitectClass = get_architect_from_type(architect_type) - params = get_extra_argument_dicts(ArchitectClass) - return jsonify({"success": True, "options": params}) - - -@api.route("/unit//accept", methods=["POST"]) -def accept_unit(unit_id): - return jsonify({"success": True}) - pass - - -@api.route("/unit//reject", methods=["POST"]) -def reject_unit(unit_id): - return jsonify({"success": True}) - pass - - -@api.route("/unit//softBlock", methods=["POST"]) -def soft_block_unit(unit_id): - return jsonify({"success": True}) - pass - - -@api.route("/unit//hardBlock", methods=["POST"]) -def hard_block_unit(unit_id): - return jsonify({"success": True}) - pass - - -@api.route("/error", defaults={"status_code": "501"}) -@api.route("/error/") -def intentional_error(status_code): - """ - A helper endpoint to test out cases in the UI where an error occurs. - """ - raise InvalidUsage("An error occured", status_code=int(status_code)) - - -class InvalidUsage(Exception): - status_code = 400 - - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self): - rv = dict(self.payload or ()) - rv["message"] = self.message - return rv - - -@api.errorhandler(InvalidUsage) -def handle_invalid_usage(error): - response = jsonify(error.to_dict()) - response.status_code = error.status_code - return response diff --git a/mephisto/client/review_app/client/src/components/Errors/Errors.css b/mephisto/client/review_app/client/src/components/Errors/Errors.css index ac20578fb..3c2e72d77 100644 --- a/mephisto/client/review_app/client/src/components/Errors/Errors.css +++ b/mephisto/client/review_app/client/src/components/Errors/Errors.css @@ -8,4 +8,5 @@ position: fixed; top: 0; right: 0; + z-index: 9999; } diff --git a/mephisto/client/review_app/client/src/components/mnist_core_components_copied.js b/mephisto/client/review_app/client/src/components/mnist_core_components_copied.js deleted file mode 100644 index b6866416d..000000000 --- a/mephisto/client/review_app/client/src/components/mnist_core_components_copied.js +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2017-present, Facebook, Inc. - * All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -import React, { Fragment } from "react"; -import CanvasDraw from "react-canvas-draw"; - -function LoadingScreen() { - return Loading...; -} - -function Directions({ children }) { - return ( -
-
-
-

{children}

-
-
-
- ); -} - -function AnnotationCanvas({ onUpdate, classifyDigit, index }) { - const [currentAnnotation, setCurrentAnnotation] = React.useState(null); - const [trueAnnotation, setTrueAnnotation] = React.useState(""); - const [isCorrect, setIsCorrect] = React.useState(null); - - const querying = React.useRef(false); - const changed = React.useRef(false); - const canvasRef = React.useRef(); - - function triggerUpdate() { - onUpdate({ - isCorrect, - trueAnnotation, - currentAnnotation, - imgData: canvasRef.current.getDataURL("png", false, "#FFF"), - }); - } - - React.useEffect(() => triggerUpdate(), [isCorrect, trueAnnotation]); - - function submitAndAnnotate() { - let urlData = canvasRef.current.getDataURL("png", false, "#FFF"); - querying.current = true; - classifyDigit({ urlData }).then((res) => { - setCurrentAnnotation(res["digit_prediction"]); - triggerUpdate(urlData); - querying.current = false; - if (changed.current === true) { - // If it's changed since we last ran, rerun! - changed.current = false; - submitAndAnnotate(); - } - }); - } - - const canvas = ( - { - if (!querying.current) { - submitAndAnnotate(); - } else { - changed.current = true; // Query once last one comes in - } - }} - canvasWidth={250} - canvasHeight={250} - brushColor={"#000"} - brushRadius={18} - hideInterface={true} - ref={(canvasDraw) => (canvasRef.current = canvasDraw)} - /> - ); - - return ( -
-
{canvas}
- -
- - Current Annotation: {currentAnnotation} - -
- Annotation Correct?{" "} - setIsCorrect(!isCorrect)} - /> - {!isCorrect && ( - -
- Corrected Annotation: -
- setTrueAnnotation(evt.target.value)} - /> - - )} -
- ); -} - -function Instructions({ taskData }) { - return ( -
-

MNIST Model Evaluator

-

- {taskData?.isScreeningUnit - ? "Screening Unit:" - : "To submit this task, you'll need to draw 3 (single) digits in the boxes below. Our model will try to provide an annotation for each."} -

-

- {taskData?.isScreeningUnit - ? 'To submit this task you will have to correctly draw the number 3 in the box below and check the "Annotation Correct" checkbox' - : "You can confirm or reject each of the annotations. Provide a correction if the annotation is wrong."} -

-
- ); -} - -export function TaskFrontend({ taskData, classifyDigit, handleSubmit }) { - const NUM_ANNOTATIONS = taskData.isScreeningUnit ? 1 : 3; - const [annotations, updateAnnotations] = React.useReducer( - (currentAnnotation, { updateIdx, updatedAnnotation }) => { - return currentAnnotation.map((val, idx) => - idx == updateIdx ? updatedAnnotation : val - ); - }, - Array(NUM_ANNOTATIONS).fill({ - currentAnnotation: null, - trueAnnotation: null, - isCorrect: null, - }) - ); - let canSubmit = - annotations.filter((a) => a.isCorrect === true || a.trueAnnotation !== "") - .length == NUM_ANNOTATIONS; - - return ( -
- -
- {annotations.map((_d, idx) => ( - - updateAnnotations({ - updateIdx: idx, - updatedAnnotation: annotation, - }) - } - /> - ))} -
-
- - -
- ); -} diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.css b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.css index 2dc3eae5b..544ed63d9 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.css +++ b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.css @@ -10,10 +10,19 @@ border: 1px solid black; } -.review-form .second-line { +.review-form .second-line, +.review-form .third-line { margin-top: 10px; box-sizing: border-box; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; padding-left: 25px; } + +.review-form .third-line { + margin-top: 10px; +} + +.new-qualification-name-button { + width: 100%; +} diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx index 33c8324ed..2b18e9db4 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx @@ -7,8 +7,8 @@ import { ReviewType } from "consts/review"; import * as React from "react"; import { useEffect } from "react"; -import { Col, Form, Row } from "react-bootstrap"; -import { getQualifications } from "requests/qualifications"; +import { Button, Col, Form, Row } from "react-bootstrap"; +import { getQualifications, postQualification } from "requests/qualifications"; import "./ModalForm.css"; const range = (start, end) => Array.from(Array(end + 1).keys()).slice(start); @@ -24,7 +24,8 @@ function ModalForm(props: ModalFormProps) { const [qualifications, setQualifications] = React.useState< Array >(null); - const [loading, setLoading] = React.useState(false); + const [getQualificationsloading, setGetQualificationsloading] = React.useState(false); + const [, setCreateQualificationLoading] = React.useState(false); const onChangeAssign = (value: boolean) => { let prevFormData: FormType = Object(props.data.form); @@ -38,9 +39,16 @@ function ModalForm(props: ModalFormProps) { props.setData({ ...props.data, form: prevFormData }); }; - const onChangeAssignQualification = (id: string) => { + const onChangeAssignQualification = (value: string) => { let prevFormData: FormType = Object(props.data.form); - prevFormData.qualification = Number(id); + + if (value === "+") { + prevFormData.showNewQualification = true; + prevFormData.newQualificationValue = ""; + } else { + prevFormData.qualification = Number(value); + } + props.setData({ ...props.data, form: prevFormData }); }; @@ -84,25 +92,64 @@ function ModalForm(props: ModalFormProps) { props.setData({ ...props.data, form: prevFormData }); }; + const onChangeNewQualificationValue = (value: string) => { + let prevFormData: FormType = Object(props.data.form); + prevFormData.newQualificationValue = value; + props.setData({ ...props.data, form: prevFormData }); + }; + + const onClickAddNewQualification = (value: string) => { + createNewQualification(value); + }; + const onError = (errorResponse: ErrorResponseType | null) => { if (errorResponse) { props.setErrors((oldErrors) => [...oldErrors, ...[errorResponse.error]]); } }; + const onCreateNewQualificationSuccess = () => { + // Clear input + let prevFormData: FormType = Object(props.data.form); + prevFormData.newQualificationValue = ""; + prevFormData.showNewQualification = false; + props.setData({ ...props.data, form: prevFormData }); + + // Update select with Qualifications + requestQualifications(); + }; + + const requestQualifications = () => { + let params; + if (props.data.type === ReviewType.REJECT) { + params = { worker_id: props.workerId }; + } + + getQualifications( + setQualifications, + setGetQualificationsloading, + onError, + params, + ); + }; + + const createNewQualification = (name: string) => { + postQualification( + onCreateNewQualificationSuccess, + setCreateQualificationLoading, + onError, + {name: name}, + ); + }; + // Effiects useEffect(() => { if (qualifications === null) { - let params; - if (props.data.type === ReviewType.REJECT) { - params = { worker_id: props.workerId }; - } - - getQualifications(setQualifications, setLoading, onError, params); + requestQualifications(); } }, []); - if (loading) { + if (getQualificationsloading) { return; } @@ -126,7 +173,7 @@ function ModalForm(props: ModalFormProps) { } /> - {props.data.form.checkboxAssignQualification && ( + {props.data.form.checkboxAssignQualification && (<> onChangeAssignQualification(e.target.value)} > + {qualifications && qualifications.map((q: QualificationType) => { return ( @@ -161,7 +209,32 @@ function ModalForm(props: ModalFormProps) { - )} + {props.data.form.showNewQualification && ( + + + onChangeNewQualificationValue(e.target.value)} + /> + + + + + + )} + )} )} diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx index 7cf4ea291..ac0037541 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx @@ -4,13 +4,11 @@ * LICENSE file in the root directory of this source tree. */ -// TODO: Find the way to import it dynamically -// import { TaskFrontend } from 'components/mnist_core_components_copied'; import { ReviewType } from "consts/review"; import cloneDeep from "lodash/cloneDeep"; import * as React from "react"; import { useEffect } from "react"; -import { Button, Spinner, Table } from "react-bootstrap"; +import { Button, Spinner } from "react-bootstrap"; import JSONPretty from "react-json-pretty"; import { useNavigate, useParams } from "react-router-dom"; import { diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/modalData.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/modalData.tsx index cbbef246e..c8a72b252 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/modalData.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/modalData.tsx @@ -12,13 +12,15 @@ export const APPROVE_MODAL_DATA_STATE: ModalDataType = { buttonCancel: "Cancel", buttonSubmit: "Approve", form: { + bonus: null, checkboxAssignQualification: false, - checkboxReviewNote: false, checkboxGiveBonus: false, - reviewNote: "", + checkboxReviewNote: false, + newQualificationValue: "", qualification: null, qualificationValue: 1, - bonus: null, + reviewNote: "", + showNewQualification: false, }, title: "Approve Unit", type: ReviewType.APPROVE, @@ -30,12 +32,14 @@ export const SOFT_REJECT_MODAL_DATA_STATE: ModalDataType = { buttonCancel: "Cancel", buttonSubmit: "Soft-Reject", form: { + bonus: null, checkboxAssignQualification: false, checkboxReviewNote: false, - reviewNote: "", + newQualificationValue: "", qualification: null, qualificationValue: 1, - bonus: null, + reviewNote: "", + showNewQualification: false, }, title: "Soft-Reject Unit", type: ReviewType.SOFT_REJECT, diff --git a/mephisto/client/review_app/client/src/types/reviewModal.d.ts b/mephisto/client/review_app/client/src/types/reviewModal.d.ts index 95fd3280b..7e35ca411 100644 --- a/mephisto/client/review_app/client/src/types/reviewModal.d.ts +++ b/mephisto/client/review_app/client/src/types/reviewModal.d.ts @@ -5,15 +5,17 @@ */ type FormType = { + bonus: number | null; checkboxAssignQualification?: boolean; checkboxBanWorker?: boolean; - checkboxReviewNote: boolean; checkboxGiveBonus?: boolean; + checkboxReviewNote: boolean; checkboxUnassignQualification?: boolean; - reviewNote: string; + newQualificationValue?: string; qualification: number | null; qualificationValue: number; - bonus: number | null; + reviewNote: string; + showNewQualification?: boolean; }; type ModalDataType = { diff --git a/mephisto/client/review_app/server/api/views/qualifications_view.py b/mephisto/client/review_app/server/api/views/qualifications_view.py index 5141327bb..ad02a0fc3 100644 --- a/mephisto/client/review_app/server/api/views/qualifications_view.py +++ b/mephisto/client/review_app/server/api/views/qualifications_view.py @@ -101,7 +101,7 @@ def post(self) -> dict: db_qualifications: List[Qualification] = app.db.find_qualifications(qualification_name) if db_qualifications: - raise BadRequest(f'Qualifications with name "{qualification_name}" already exists.') + raise BadRequest(f'Qualification with name "{qualification_name}" already exists.') db_qualification_id: str = app.db.make_qualification(qualification_name) db_qualification: StringIDRow = app.db.get_qualification(db_qualification_id)