From 11c129abe302160749e7fd085406e7f539bb6084 Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Fri, 19 Jan 2024 16:10:37 -0500 Subject: [PATCH 1/3] [Task Review App] Enabled worker bonusing and messaging --- .../abstractions/providers/mock/mock_agent.py | 14 +++-- .../providers/mock/mock_worker.py | 22 ++++++-- .../providers/mturk/mturk_agent.py | 32 +++++------ .../providers/mturk/mturk_worker.py | 54 +++++++++++++------ .../providers/prolific/api/messages.py | 8 ++- .../providers/prolific/prolific_agent.py | 9 ++-- .../providers/prolific/prolific_utils.py | 12 ++++- .../providers/prolific/prolific_worker.py | 24 ++++++++- mephisto/client/review_app/README.md | 20 +++---- mephisto/client/review_app/client/README.md | 10 ++-- .../pages/TaskPage/ModalForm/ModalForm.tsx | 5 +- .../client/src/pages/TaskPage/TaskPage.tsx | 2 +- .../server/api/views/units_approve_view.py | 9 +++- .../server/api/views/units_reject_view.py | 2 +- .../api/views/units_soft_reject_view.py | 2 +- mephisto/data_model/agent.py | 8 +-- mephisto/data_model/worker.py | 25 ++++++--- .../providers/prolific/test_prolific_utils.py | 24 +++++++++ 18 files changed, 196 insertions(+), 86 deletions(-) diff --git a/mephisto/abstractions/providers/mock/mock_agent.py b/mephisto/abstractions/providers/mock/mock_agent.py index e613f1922..fc955b8b0 100644 --- a/mephisto/abstractions/providers/mock/mock_agent.py +++ b/mephisto/abstractions/providers/mock/mock_agent.py @@ -4,17 +4,21 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.agent import Agent +from typing import Any +from typing import Dict +from typing import Mapping +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + from mephisto.abstractions.blueprint import AgentState from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE - -from typing import List, Optional, Tuple, Dict, Mapping, Any, TYPE_CHECKING +from mephisto.data_model.agent import Agent if TYPE_CHECKING: from mephisto.data_model.unit import Unit from mephisto.abstractions.database import MephistoDB from mephisto.data_model.worker import Worker - from mephisto.data_model.packet import Packet from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore @@ -76,7 +80,7 @@ def get_live_update(self, timeout=None) -> Optional[Dict[str, Any]]: def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """ diff --git a/mephisto/abstractions/providers/mock/mock_worker.py b/mephisto/abstractions/providers/mock/mock_worker.py index c407de64f..02cd7e814 100644 --- a/mephisto/abstractions/providers/mock/mock_worker.py +++ b/mephisto/abstractions/providers/mock/mock_worker.py @@ -4,22 +4,30 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.worker import Worker +from typing import Any +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING + from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE -from typing import List, Optional, Tuple, Dict, Mapping, Type, Any, TYPE_CHECKING +from mephisto.data_model.worker import Worker +from mephisto.utils.logger_core import get_logger if TYPE_CHECKING: from mephisto.abstractions.database import MephistoDB from mephisto.data_model.task_run import TaskRun from mephisto.data_model.unit import Unit - from mephisto.data_model.agent import Agent from mephisto.data_model.requester import Requester from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore +logger = get_logger(name=__name__) + class MockWorker(Worker): """ - This class represents an individual - namely a person. It maintains components of ongoing identity for a user. + This class represents an individual - namely a person. + It maintains components of ongoing identity for a user. """ def __init__( @@ -36,6 +44,7 @@ def bonus_worker( self, amount: float, reason: str, unit: Optional["Unit"] = None ) -> Tuple[bool, str]: """Bonus this worker for work any reason. Return success of bonus""" + logger.debug(f"Mock paying bonus to worker. Amount: {amount}. Reason: '{reason}") return True, "" def block_worker( @@ -61,6 +70,11 @@ def is_eligible(self, task_run: "TaskRun") -> bool: """Determine if this worker is eligible for the given task run""" return True + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + logger.debug(f"Mock sending feedback message to worker: '{text}'. Unit: {unit}") + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": return MockWorker._register_worker(db, worker_id + "_sandbox", PROVIDER_TYPE) diff --git a/mephisto/abstractions/providers/mturk/mturk_agent.py b/mephisto/abstractions/providers/mturk/mturk_agent.py index dca107d38..72d3eac86 100644 --- a/mephisto/abstractions/providers/mturk/mturk_agent.py +++ b/mephisto/abstractions/providers/mturk/mturk_agent.py @@ -4,25 +4,25 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.agent import Agent -from mephisto.abstractions.blueprint import AgentState -from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE -from mephisto.abstractions.providers.mturk.mturk_utils import ( - approve_work, - reject_work, - get_assignment, - get_assignments_for_hit, -) - -import xmltodict # type: ignore import json +from typing import Any +from typing import cast +from typing import Dict +from typing import Mapping +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union -from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING +import xmltodict # type: ignore +from mephisto.abstractions.blueprint import AgentState +from mephisto.abstractions.providers.mturk.mturk_utils import approve_work +from mephisto.abstractions.providers.mturk.mturk_utils import get_assignments_for_hit +from mephisto.abstractions.providers.mturk.mturk_utils import reject_work +from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE +from mephisto.data_model.agent import Agent from mephisto.utils.logger_core import get_logger -logger = get_logger(name=__name__) - if TYPE_CHECKING: from mephisto.data_model.unit import Unit from mephisto.abstractions.database import MephistoDB @@ -31,6 +31,8 @@ from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore +logger = get_logger(name=__name__) + class MTurkAgent(Agent): """ @@ -104,7 +106,7 @@ def attempt_to_reconcile_submitted_data(self, mturk_hit_id: str): def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """Approve the work done on this specific Unit""" diff --git a/mephisto/abstractions/providers/mturk/mturk_worker.py b/mephisto/abstractions/providers/mturk/mturk_worker.py index e5326950b..68998f19c 100644 --- a/mephisto/abstractions/providers/mturk/mturk_worker.py +++ b/mephisto/abstractions/providers/mturk/mturk_worker.py @@ -4,32 +4,33 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.worker import Worker -from mephisto.data_model.requester import Requester -from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE -from mephisto.abstractions.providers.mturk.mturk_utils import ( - pay_bonus, - block_worker, - unblock_worker, - is_worker_blocked, - give_worker_qualification, - remove_worker_qualification, -) -from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester - +from typing import Any +from typing import cast +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING from uuid import uuid4 -from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING +from mephisto.abstractions.providers.mturk.mturk_utils import block_worker +from mephisto.abstractions.providers.mturk.mturk_utils import email_worker +from mephisto.abstractions.providers.mturk.mturk_utils import give_worker_qualification +from mephisto.abstractions.providers.mturk.mturk_utils import is_worker_blocked +from mephisto.abstractions.providers.mturk.mturk_utils import pay_bonus +from mephisto.abstractions.providers.mturk.mturk_utils import remove_worker_qualification +from mephisto.abstractions.providers.mturk.mturk_utils import unblock_worker +from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE +from mephisto.data_model.requester import Requester +from mephisto.data_model.worker import Worker +from mephisto.utils.logger_core import get_logger if TYPE_CHECKING: from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore from mephisto.abstractions.database import MephistoDB from mephisto.data_model.task_run import TaskRun from mephisto.data_model.unit import Unit - from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester -from mephisto.utils.logger_core import get_logger logger = get_logger(name=__name__) @@ -179,6 +180,27 @@ def is_eligible(self, task_run: "TaskRun") -> bool: """ return True + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + requester = cast( + "MTurkRequester", + self.db.find_requesters(provider_type=self.provider_type)[-1], + ) + + assert isinstance(requester, MTurkRequester), "Must be an MTurk requester" + + client = self._get_client(requester._requester_name) + task_name = unit.get_task_run().get_task().task_name + + email_worker( + client=client, + worker_id=self.get_mturk_worker_id(), + subject=f'Feedback for your Mturk task "{task_name}"', + message_text=text, + ) + + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": return MTurkWorker._register_worker(db, worker_id, PROVIDER_TYPE) diff --git a/mephisto/abstractions/providers/prolific/api/messages.py b/mephisto/abstractions/providers/prolific/api/messages.py index ebf27eed9..cb4bf1454 100644 --- a/mephisto/abstractions/providers/prolific/api/messages.py +++ b/mephisto/abstractions/providers/prolific/api/messages.py @@ -52,9 +52,13 @@ def list_unread(cls) -> List[Message]: return messages @classmethod - def send(cls, **data) -> Message: + def send(cls, study_id: str, recipient_id: str, text: str) -> Message: """Send a message to a participant or another researcher""" - message = Message(**data) + message = Message( + body=text, + recipient_id=recipient_id, + study_id=study_id, + ) message.validate() response_json = cls.post(cls.list_api_endpoint, params=message.to_dict()) return Message(**response_json) diff --git a/mephisto/abstractions/providers/prolific/prolific_agent.py b/mephisto/abstractions/providers/prolific/prolific_agent.py index a97a8d78c..ba504dd63 100644 --- a/mephisto/abstractions/providers/prolific/prolific_agent.py +++ b/mephisto/abstractions/providers/prolific/prolific_agent.py @@ -10,14 +10,15 @@ from typing import Mapping from typing import Optional from typing import TYPE_CHECKING +from typing import Union from mephisto.abstractions.blueprint import AgentState from mephisto.abstractions.providers.prolific import prolific_utils +from mephisto.abstractions.providers.prolific.api.client import ProlificClient +from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE from mephisto.data_model.agent import Agent from mephisto.utils.logger_core import get_logger -from mephisto.abstractions.providers.prolific.api.client import ProlificClient -from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus if TYPE_CHECKING: from mephisto.abstractions.providers.prolific.prolific_datastore import ProlificDatastore @@ -103,7 +104,7 @@ def new_from_provider_data( def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """Approve the work done on this specific Unit""" @@ -139,7 +140,7 @@ def approve_work( worker_id=unit.worker_id, status=AgentState.STATUS_APPROVED, review_note=review_note, - bonus=bonus, + bonus=str(bonus), ) def soft_reject_work(self, review_note: Optional[str] = None) -> None: diff --git a/mephisto/abstractions/providers/prolific/prolific_utils.py b/mephisto/abstractions/providers/prolific/prolific_utils.py index 1d85cfa6c..0b9593a58 100644 --- a/mephisto/abstractions/providers/prolific/prolific_utils.py +++ b/mephisto/abstractions/providers/prolific/prolific_utils.py @@ -28,6 +28,7 @@ from .api.base_api_resource import CREDENTIALS_CONFIG_PATH from .api.client import ProlificClient from .api.data_models import BonusPayments +from .api.data_models import Message from .api.data_models import Participant from .api.data_models import ParticipantGroup from .api.data_models import Project @@ -596,7 +597,7 @@ def pay_bonus( client: ProlificClient, task_run_config: "DictConfig", worker_id: str, - bonus_amount: int, # in cents + bonus_amount: Union[int, float], # in cents study_id: str, *args, **kwargs, @@ -798,3 +799,12 @@ def reject_work( ) return None + + +def send_message(client: ProlificClient, study_id: str, participant_id: str, text: str) -> Message: + try: + message: Message = client.Messages.send(study_id, participant_id, text) + except (ProlificException, ValidationError): + logger.exception(f'Could not send message to participant "{participant_id}"') + raise + return message diff --git a/mephisto/abstractions/providers/prolific/prolific_worker.py b/mephisto/abstractions/providers/prolific/prolific_worker.py index d0e88a8f9..20c55b58f 100644 --- a/mephisto/abstractions/providers/prolific/prolific_worker.py +++ b/mephisto/abstractions/providers/prolific/prolific_worker.py @@ -13,8 +13,6 @@ from typing import Tuple from typing import TYPE_CHECKING -from omegaconf import DictConfig - from mephisto.abstractions.providers.prolific import prolific_utils from mephisto.abstractions.providers.prolific.api.client import ProlificClient from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE @@ -298,6 +296,28 @@ def revoke_crowd_qualification(self, qualification_name: str) -> None: return None + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + requester = cast( + "ProlificRequester", + self.db.find_requesters(provider_type=self.provider_type)[-1], + ) + + assert isinstance(requester, ProlificRequester), "Must be an Prolific requester" + + client = self._get_client(requester.requester_name) + datastore_unit = self.datastore.get_unit(unit.db_id) + prolific_study_id = datastore_unit["prolific_study_id"] + + prolific_utils.send_message( + client=client, + study_id=prolific_study_id, + participant_id=self.get_prolific_participant_id(), + text=text, + ) + + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": new_worker = ProlificWorker._register_worker(db, worker_id, PROVIDER_TYPE) diff --git a/mephisto/client/review_app/README.md b/mephisto/client/review_app/README.md index bd6d2e4d1..aefcc9c9a 100644 --- a/mephisto/client/review_app/README.md +++ b/mephisto/client/review_app/README.md @@ -63,11 +63,11 @@ _The "review" version will be shown to reviewer inside an iframe, while "regular 1. Create `review_app` subdirectory of your app, next to the main `app` subdirectory. Its content should include: - Main JS file `review.js` for ReviewApp - - Example: [review.js](../../../examples/remote_procedure/mnist/webapp/src/review.js) + - Example: [review.js](/examples/remote_procedure/mnist/webapp/src/review.js) - ReviewApp `reviewapp.jsx` - - Example: [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) - Separate Webpack config `webpack.config.review.js` - - Example: [webpack.config.review.js](../../../examples/remote_procedure/mnist/webapp/webpack.config.review.js) + - Example: [webpack.config.review.js](/examples/remote_procedure/mnist/webapp/webpack.config.review.js) 2. Specify in your Hydra YAML, under mephisto.blueprint section: ```json @@ -78,18 +78,18 @@ task_source_review: ${task_dir}/webapp/build/bundle.review.js ```json "build:review": "webpack --config=webpack.config.review.js --mode development" ``` -Example: [package.json](../../../examples/remote_procedure/mnist/webapp/package.json) +Example: [package.json](/examples/remote_procedure/mnist/webapp/package.json) 4. Build this "review" bundle by running `npm run build:review` from directory with `package.json`. 5. This `reviewapp.jsx` must satisfy 3 requirements, to interface with TaskReview: - Render "review" task version only upon receiving messages with Task data: - - Example: comment #1 in [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: comment #1 in [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) - Send messages with displayed "review" page height (to resize containing iframe and avoid scrollbars) - - Example: comment #2 in [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: comment #2 in [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) `
` - Rendered component must always return reference to `appRef`, like so `
` - - Example: comment #3 in [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: comment #3 in [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) --- @@ -99,6 +99,6 @@ A quick overview of how TaskReview looks like. _Note that if your Task doesn't provide a custom layout for TaskReview App, in Submission view you'll see a generic display of JSON data._ -![List of tasks](screenshots/task_list.png) -![Generic submission view](screenshots/generic_submission_view.png) -![Submission approval](screenshots/approve_unit.png) +![List of tasks](./screenshots/task_list.png) +![Generic submission view](./screenshots/generic_submission_view.png) +![Submission approval](./screenshots/approve_unit.png) diff --git a/mephisto/client/review_app/client/README.md b/mephisto/client/review_app/client/README.md index e1bb6aa2a..612e7af0b 100644 --- a/mephisto/client/review_app/client/README.md +++ b/mephisto/client/review_app/client/README.md @@ -10,18 +10,18 @@ - User selects a task - We get list of available qualifications from `GET /qualifications` - We pull all unit-worker id pairs from `GET /tasks/{id}/worker-units-ids` - - *Due to the need to randomly shuffle units grouped by a worker (to mitigate reviewers bias, etc) we're implementing client-side pagination - client gets full list of all ids, creates a page of unit ids, and then pulls data for those specific units.* + - *Due to the need to randomly shuffle units grouped by a worker (to mitigate reviewers bias, etc) we're implementing client-side pagination - client gets full list of all ids, creates a page of unit ids, and then pulls data for those specific units.* - We initiate units review by worker: - Group units by worker - Sort workers by number of their units fewest units go first) - Pick them for review one-by-one - For each worker: - - We pull units by ids from `GET /units?unit_ids=[...]` - - We sort units by `creation_date` and pick them for review one-by-one - - For each reviewed unit: + - We pull units by ids from `GET /units?unit_ids=[...]` + - We sort units by `creation_date` and pick them for review one-by-one + - For each reviewed unit: - We pull unit details from `GET /units/details?unit_ids=[...]` - We pull current stats from `GET /stats` (for entire task and for worker within the task) - We render unit's review representation in an iframe - User can choose to reject/accept unit, grant/revoke qualification, and block the worker - When all units are reviewed, user is redirected to "Tasks" page and clicks "Download" button for the reviewed Task - - We pull Task data from `GET /tasks//export-results.json` endpoint + - We pull Task data from `GET /tasks//export-results.json` endpoint diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx index a8601fbea..f8832acb6 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/ModalForm/ModalForm.tsx @@ -11,9 +11,8 @@ import { Button, Col, Form, Row } from "react-bootstrap"; import { getQualifications, postQualification } from "requests/qualifications"; import "./ModalForm.css"; -// TODO(#1058): [Review APP] Implement back-end for this functionality -const BONUS_FOR_WORKER_ENABLED = false; -const FEEDBACK_FOR_WORKER_ENABLED = false; +const BONUS_FOR_WORKER_ENABLED = true; +const FEEDBACK_FOR_WORKER_ENABLED = true; const range = (start, end) => Array.from(Array(end + 1).keys()).slice(start); diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx index 51c7b3a9e..e2593f5be 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx @@ -296,7 +296,7 @@ function TaskPage(props: PropsType) { setLoading, onError, { - bonus: modalData.form.checkboxGiveBonus ? modalData.form.bonus : null, + bonus: modalData.form.checkboxGiveBonus ? Number(modalData.form.bonus) : null, review_note: modalData.form.checkboxReviewNote ? modalData.form.reviewNote : null, diff --git a/mephisto/client/review_app/server/api/views/units_approve_view.py b/mephisto/client/review_app/server/api/views/units_approve_view.py index ae6990d5a..216261fcd 100644 --- a/mephisto/client/review_app/server/api/views/units_approve_view.py +++ b/mephisto/client/review_app/server/api/views/units_approve_view.py @@ -5,6 +5,7 @@ # LICENSE file in the root directory of this source tree. from typing import Optional +from typing import Union from flask import current_app as app from flask import request @@ -21,8 +22,9 @@ def post(self) -> dict: data: dict = request.json unit_ids: Optional[str] = data.get("unit_ids") if data else None - review_note = data.get("review_note") if data else None # Optional - bonus = data.get("bonus") if data else None # Optional + review_note: Optional[str] = data.get("review_note") if data else None + bonus: Optional[Union[int, float]] = data.get("bonus") if data else None + send_to_worker: Optional[bool] = data.get("send_to_worker", False) if data else False # Validate params if not unit_ids: @@ -63,4 +65,7 @@ def post(self) -> dict: app.logger.exception("Could not pay bonus. Unexpected error") return {} + if review_note and send_to_worker: + worker.send_feedback_message(text=review_note, unit=unit) + return {} diff --git a/mephisto/client/review_app/server/api/views/units_reject_view.py b/mephisto/client/review_app/server/api/views/units_reject_view.py index ba5567053..1c8c4d1c4 100644 --- a/mephisto/client/review_app/server/api/views/units_reject_view.py +++ b/mephisto/client/review_app/server/api/views/units_reject_view.py @@ -20,7 +20,7 @@ def post(self) -> dict: data: dict = request.json unit_ids: Optional[str] = data and data.get("unit_ids") - review_note = data and data.get("review_note") # Optional + review_note: Optional[str] = data.get("review_note") if data else None # Validate params if not unit_ids: diff --git a/mephisto/client/review_app/server/api/views/units_soft_reject_view.py b/mephisto/client/review_app/server/api/views/units_soft_reject_view.py index f91a57114..f80feddfd 100644 --- a/mephisto/client/review_app/server/api/views/units_soft_reject_view.py +++ b/mephisto/client/review_app/server/api/views/units_soft_reject_view.py @@ -20,7 +20,7 @@ def post(self) -> dict: data: dict = request.json unit_ids: Optional[str] = data and data.get("unit_ids") - review_note = data and data.get("review_note") # Optional + review_note: Optional[str] = data.get("review_note") if data else None # Validate params if not unit_ids: diff --git a/mephisto/data_model/agent.py b/mephisto/data_model/agent.py index 81afa2044..390fadf81 100644 --- a/mephisto/data_model/agent.py +++ b/mephisto/data_model/agent.py @@ -5,11 +5,8 @@ # LICENSE file in the root directory of this source tree. from __future__ import annotations -import csv -from genericpath import exists import os -from pathlib import Path import threading from queue import Queue from uuid import uuid4 @@ -29,7 +26,7 @@ AgentShutdownError, ) -from typing import Optional, Mapping, Dict, Any, cast, TYPE_CHECKING +from typing import Optional, Mapping, Dict, Any, cast, TYPE_CHECKING, Union try: from detoxify import Detoxify @@ -42,7 +39,6 @@ from mephisto.data_model.unit import Unit from mephisto.data_model.assignment import Assignment from mephisto.abstractions.database import MephistoDB - from mephisto.data_model.packet import Packet from mephisto.data_model.task import Task from mephisto.data_model.task_run import TaskRun from mephisto.operations.datatypes import LiveTaskRun @@ -560,7 +556,7 @@ def get_status(self) -> str: def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """Approve the work done on this agent's specific Unit""" diff --git a/mephisto/data_model/worker.py b/mephisto/data_model/worker.py index 2b1f9bb7f..3caed2d1a 100644 --- a/mephisto/data_model/worker.py +++ b/mephisto/data_model/worker.py @@ -4,14 +4,20 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING + from mephisto.abstractions.blueprint import AgentState -from mephisto.data_model._db_backed_meta import ( - MephistoDBBackedABCMeta, - MephistoDataModelComponentMixin, -) -from typing import Any, List, Optional, Mapping, Tuple, Dict, Type, Tuple, TYPE_CHECKING +from mephisto.data_model._db_backed_meta import MephistoDataModelComponentMixin +from mephisto.data_model._db_backed_meta import MephistoDBBackedABCMeta from mephisto.utils.logger_core import get_logger logger = get_logger(name=__name__) @@ -24,7 +30,6 @@ from mephisto.data_model.requester import Requester from mephisto.data_model.task_run import TaskRun from mephisto.data_model.qualification import GrantedQualification - from argparse import _ArgumentGroup as ArgumentGroup @dataclass @@ -271,6 +276,10 @@ def register(self, args: Optional[Dict[str, str]] = None) -> None: """Register this worker with the crowdprovider, if necessary""" pass + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + raise NotImplementedError() + @staticmethod def new(db: "MephistoDB", worker_name: str) -> "Worker": """ diff --git a/test/abstractions/providers/prolific/test_prolific_utils.py b/test/abstractions/providers/prolific/test_prolific_utils.py index 171d837d2..a75469137 100644 --- a/test/abstractions/providers/prolific/test_prolific_utils.py +++ b/test/abstractions/providers/prolific/test_prolific_utils.py @@ -64,6 +64,7 @@ from mephisto.abstractions.providers.prolific.prolific_utils import publish_study from mephisto.abstractions.providers.prolific.prolific_utils import reject_work from mephisto.abstractions.providers.prolific.prolific_utils import remove_worker_qualification +from mephisto.abstractions.providers.prolific.prolific_utils import send_message from mephisto.abstractions.providers.prolific.prolific_utils import setup_credentials from mephisto.abstractions.providers.prolific.prolific_utils import stop_study from mephisto.abstractions.providers.prolific.prolific_utils import unblock_worker @@ -1482,6 +1483,29 @@ def test_reject_work_exception(self, mock_get_submission, mock_reject, *args): self.assertEqual(cm.exception.message, exception_message) + @patch(f"{API_PATH}.messages.Messages.send") + def test_send_message_success(self, *args): + study_id = "test" + participant_id = "test2" + text = "test3" + + result = send_message(self.client, study_id, participant_id, text) + + self.assertIsNone(result) + + @patch(f"{API_PATH}.messages.Messages.send") + def test_send_message_exception(self, mock_send, *args): + study_id = "test" + participant_id = "test2" + text = "test3" + + exception_message = "Error" + mock_send.side_effect = ProlificRequestError(exception_message) + with self.assertRaises(ProlificRequestError) as cm: + send_message(self.client, study_id, participant_id, text) + + self.assertEqual(cm.exception.message, exception_message) + if __name__ == "__main__": unittest.main() From bb28053af04950c65b087c1a316fadc8ca2e99f4 Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Wed, 24 Jan 2024 09:17:38 -0500 Subject: [PATCH 2/3] Mephisto example projects launching instruction --- examples/README.md | 69 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 58f14e96e..b316ca0f6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,4 +5,71 @@ --> # Examples -The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks. \ No newline at end of file +The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks. + +Mephisto Tasks can be launched (each run is called TaskRun) with a single `docker-compose` command (you will need to have Docker [installed](https://docs.docker.com/engine/install/).) + +Let's launch Mephisto example tasks, starting from the easiest one + +--- + +#### 1. Simple HTML-based task + +A simple project with HTML-based UI task template [simple_static_task](/examples/simple_static_task) + +- Default config file: [/examples/simple_static_task/hydra_configs/conf/example.yaml] +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + python /mephisto/examples/simple_static_task/static_test_script.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1) +- Browser page should display an image, instruction, select and file inputs, and a submit button. + +--- + +#### 2. Simple React-based task + +A simple project with React-based UI task template [static_react_task](/examples/static_react_task) + +- Default config file: [example.yaml](/examples/static_react_task/hydra_configs/conf/example.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + python /mephisto/examples/static_react_task/run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display an instruction line and two buttons (green and red). + +--- + +#### 3. Task with dynamic input + +A more complex example featuring worker-generated dynamic input: [mnist](/examples/remote_procedure/mnist). + +- Default config file: [launch_with_local.yaml](/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + apt install curl && \ + pip install grafana torch pillow numpy && \ + mephisto metrics install && \ + python /mephisto/examples/remote_procedure/mnist/run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display instructions and a layout with 3 rectangle fields for drawing numbers with a mouse, each field having inputs at the bottom. + +--- + +# Your Mephisto project + +To read on steps for creating your own custom Mephisto task, please refer to README in the main Mephisto repo. From e56b5c064d09927d19319259c21ba3a2361ac562 Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Thu, 1 Feb 2024 16:49:44 -0500 Subject: [PATCH 3/3] Linting fixes --- .../client/review_app/client/src/pages/TaskPage/TaskPage.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx index e2593f5be..d97695352 100644 --- a/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx +++ b/mephisto/client/review_app/client/src/pages/TaskPage/TaskPage.tsx @@ -296,7 +296,9 @@ function TaskPage(props: PropsType) { setLoading, onError, { - bonus: modalData.form.checkboxGiveBonus ? Number(modalData.form.bonus) : null, + bonus: modalData.form.checkboxGiveBonus + ? Number(modalData.form.bonus) + : null, review_note: modalData.form.checkboxReviewNote ? modalData.form.reviewNote : null,