diff --git a/examples/README.md b/examples/README.md index 58f14e96e..b316ca0f6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,4 +5,71 @@ --> # Examples -The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks. \ No newline at end of file +The Mephisto example folders within contain some sample starter code and tasks to demonstrate potential workflows for setting up and working on new tasks. + +Mephisto Tasks can be launched (each run is called TaskRun) with a single `docker-compose` command (you will need to have Docker [installed](https://docs.docker.com/engine/install/).) + +Let's launch Mephisto example tasks, starting from the easiest one + +--- + +#### 1. Simple HTML-based task + +A simple project with HTML-based UI task template [simple_static_task](/examples/simple_static_task) + +- Default config file: [/examples/simple_static_task/hydra_configs/conf/example.yaml] +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + python /mephisto/examples/simple_static_task/static_test_script.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1) +- Browser page should display an image, instruction, select and file inputs, and a submit button. + +--- + +#### 2. Simple React-based task + +A simple project with React-based UI task template [static_react_task](/examples/static_react_task) + +- Default config file: [example.yaml](/examples/static_react_task/hydra_configs/conf/example.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + python /mephisto/examples/static_react_task/run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display an instruction line and two buttons (green and red). + +--- + +#### 3. Task with dynamic input + +A more complex example featuring worker-generated dynamic input: [mnist](/examples/remote_procedure/mnist). + +- Default config file: [launch_with_local.yaml](/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml). +- Launch command: + ```shell + docker-compose -f docker/docker-compose.dev.yml run \ + --build \ + --publish 3001:3000 \ + --rm mephisto_dc \ + apt install curl && \ + pip install grafana torch pillow numpy && \ + mephisto metrics install && \ + python /mephisto/examples/remote_procedure/mnist/run_task.py + ``` +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page should display instructions and a layout with 3 rectangle fields for drawing numbers with a mouse, each field having inputs at the bottom. + +--- + +# Your Mephisto project + +To read on steps for creating your own custom Mephisto task, please refer to README in the main Mephisto repo. diff --git a/mephisto/abstractions/providers/mock/mock_agent.py b/mephisto/abstractions/providers/mock/mock_agent.py index e613f1922..fc955b8b0 100644 --- a/mephisto/abstractions/providers/mock/mock_agent.py +++ b/mephisto/abstractions/providers/mock/mock_agent.py @@ -4,17 +4,21 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.agent import Agent +from typing import Any +from typing import Dict +from typing import Mapping +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + from mephisto.abstractions.blueprint import AgentState from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE - -from typing import List, Optional, Tuple, Dict, Mapping, Any, TYPE_CHECKING +from mephisto.data_model.agent import Agent if TYPE_CHECKING: from mephisto.data_model.unit import Unit from mephisto.abstractions.database import MephistoDB from mephisto.data_model.worker import Worker - from mephisto.data_model.packet import Packet from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore @@ -76,7 +80,7 @@ def get_live_update(self, timeout=None) -> Optional[Dict[str, Any]]: def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """ diff --git a/mephisto/abstractions/providers/mock/mock_worker.py b/mephisto/abstractions/providers/mock/mock_worker.py index c407de64f..02cd7e814 100644 --- a/mephisto/abstractions/providers/mock/mock_worker.py +++ b/mephisto/abstractions/providers/mock/mock_worker.py @@ -4,22 +4,30 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.worker import Worker +from typing import Any +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING + from mephisto.abstractions.providers.mock.provider_type import PROVIDER_TYPE -from typing import List, Optional, Tuple, Dict, Mapping, Type, Any, TYPE_CHECKING +from mephisto.data_model.worker import Worker +from mephisto.utils.logger_core import get_logger if TYPE_CHECKING: from mephisto.abstractions.database import MephistoDB from mephisto.data_model.task_run import TaskRun from mephisto.data_model.unit import Unit - from mephisto.data_model.agent import Agent from mephisto.data_model.requester import Requester from mephisto.abstractions.providers.mock.mock_datastore import MockDatastore +logger = get_logger(name=__name__) + class MockWorker(Worker): """ - This class represents an individual - namely a person. It maintains components of ongoing identity for a user. + This class represents an individual - namely a person. + It maintains components of ongoing identity for a user. """ def __init__( @@ -36,6 +44,7 @@ def bonus_worker( self, amount: float, reason: str, unit: Optional["Unit"] = None ) -> Tuple[bool, str]: """Bonus this worker for work any reason. Return success of bonus""" + logger.debug(f"Mock paying bonus to worker. Amount: {amount}. Reason: '{reason}") return True, "" def block_worker( @@ -61,6 +70,11 @@ def is_eligible(self, task_run: "TaskRun") -> bool: """Determine if this worker is eligible for the given task run""" return True + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + logger.debug(f"Mock sending feedback message to worker: '{text}'. Unit: {unit}") + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": return MockWorker._register_worker(db, worker_id + "_sandbox", PROVIDER_TYPE) diff --git a/mephisto/abstractions/providers/mturk/mturk_agent.py b/mephisto/abstractions/providers/mturk/mturk_agent.py index dca107d38..72d3eac86 100644 --- a/mephisto/abstractions/providers/mturk/mturk_agent.py +++ b/mephisto/abstractions/providers/mturk/mturk_agent.py @@ -4,25 +4,25 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.agent import Agent -from mephisto.abstractions.blueprint import AgentState -from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE -from mephisto.abstractions.providers.mturk.mturk_utils import ( - approve_work, - reject_work, - get_assignment, - get_assignments_for_hit, -) - -import xmltodict # type: ignore import json +from typing import Any +from typing import cast +from typing import Dict +from typing import Mapping +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union -from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING +import xmltodict # type: ignore +from mephisto.abstractions.blueprint import AgentState +from mephisto.abstractions.providers.mturk.mturk_utils import approve_work +from mephisto.abstractions.providers.mturk.mturk_utils import get_assignments_for_hit +from mephisto.abstractions.providers.mturk.mturk_utils import reject_work +from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE +from mephisto.data_model.agent import Agent from mephisto.utils.logger_core import get_logger -logger = get_logger(name=__name__) - if TYPE_CHECKING: from mephisto.data_model.unit import Unit from mephisto.abstractions.database import MephistoDB @@ -31,6 +31,8 @@ from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore +logger = get_logger(name=__name__) + class MTurkAgent(Agent): """ @@ -104,7 +106,7 @@ def attempt_to_reconcile_submitted_data(self, mturk_hit_id: str): def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """Approve the work done on this specific Unit""" diff --git a/mephisto/abstractions/providers/mturk/mturk_worker.py b/mephisto/abstractions/providers/mturk/mturk_worker.py index e5326950b..68998f19c 100644 --- a/mephisto/abstractions/providers/mturk/mturk_worker.py +++ b/mephisto/abstractions/providers/mturk/mturk_worker.py @@ -4,32 +4,33 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.data_model.worker import Worker -from mephisto.data_model.requester import Requester -from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE -from mephisto.abstractions.providers.mturk.mturk_utils import ( - pay_bonus, - block_worker, - unblock_worker, - is_worker_blocked, - give_worker_qualification, - remove_worker_qualification, -) -from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester - +from typing import Any +from typing import cast +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING from uuid import uuid4 -from typing import List, Optional, Tuple, Dict, Mapping, Any, cast, TYPE_CHECKING +from mephisto.abstractions.providers.mturk.mturk_utils import block_worker +from mephisto.abstractions.providers.mturk.mturk_utils import email_worker +from mephisto.abstractions.providers.mturk.mturk_utils import give_worker_qualification +from mephisto.abstractions.providers.mturk.mturk_utils import is_worker_blocked +from mephisto.abstractions.providers.mturk.mturk_utils import pay_bonus +from mephisto.abstractions.providers.mturk.mturk_utils import remove_worker_qualification +from mephisto.abstractions.providers.mturk.mturk_utils import unblock_worker +from mephisto.abstractions.providers.mturk.provider_type import PROVIDER_TYPE +from mephisto.data_model.requester import Requester +from mephisto.data_model.worker import Worker +from mephisto.utils.logger_core import get_logger if TYPE_CHECKING: from mephisto.abstractions.providers.mturk.mturk_datastore import MTurkDatastore from mephisto.abstractions.database import MephistoDB from mephisto.data_model.task_run import TaskRun from mephisto.data_model.unit import Unit - from mephisto.abstractions.providers.mturk.mturk_unit import MTurkUnit from mephisto.abstractions.providers.mturk.mturk_requester import MTurkRequester -from mephisto.utils.logger_core import get_logger logger = get_logger(name=__name__) @@ -179,6 +180,27 @@ def is_eligible(self, task_run: "TaskRun") -> bool: """ return True + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + requester = cast( + "MTurkRequester", + self.db.find_requesters(provider_type=self.provider_type)[-1], + ) + + assert isinstance(requester, MTurkRequester), "Must be an MTurk requester" + + client = self._get_client(requester._requester_name) + task_name = unit.get_task_run().get_task().task_name + + email_worker( + client=client, + worker_id=self.get_mturk_worker_id(), + subject=f'Feedback for your Mturk task "{task_name}"', + message_text=text, + ) + + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": return MTurkWorker._register_worker(db, worker_id, PROVIDER_TYPE) diff --git a/mephisto/abstractions/providers/prolific/api/messages.py b/mephisto/abstractions/providers/prolific/api/messages.py index ebf27eed9..cb4bf1454 100644 --- a/mephisto/abstractions/providers/prolific/api/messages.py +++ b/mephisto/abstractions/providers/prolific/api/messages.py @@ -52,9 +52,13 @@ def list_unread(cls) -> List[Message]: return messages @classmethod - def send(cls, **data) -> Message: + def send(cls, study_id: str, recipient_id: str, text: str) -> Message: """Send a message to a participant or another researcher""" - message = Message(**data) + message = Message( + body=text, + recipient_id=recipient_id, + study_id=study_id, + ) message.validate() response_json = cls.post(cls.list_api_endpoint, params=message.to_dict()) return Message(**response_json) diff --git a/mephisto/abstractions/providers/prolific/prolific_agent.py b/mephisto/abstractions/providers/prolific/prolific_agent.py index a97a8d78c..ba504dd63 100644 --- a/mephisto/abstractions/providers/prolific/prolific_agent.py +++ b/mephisto/abstractions/providers/prolific/prolific_agent.py @@ -10,14 +10,15 @@ from typing import Mapping from typing import Optional from typing import TYPE_CHECKING +from typing import Union from mephisto.abstractions.blueprint import AgentState from mephisto.abstractions.providers.prolific import prolific_utils +from mephisto.abstractions.providers.prolific.api.client import ProlificClient +from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE from mephisto.data_model.agent import Agent from mephisto.utils.logger_core import get_logger -from mephisto.abstractions.providers.prolific.api.client import ProlificClient -from mephisto.abstractions.providers.prolific.api.constants import SubmissionStatus if TYPE_CHECKING: from mephisto.abstractions.providers.prolific.prolific_datastore import ProlificDatastore @@ -103,7 +104,7 @@ def new_from_provider_data( def approve_work( self, review_note: Optional[str] = None, - bonus: Optional[str] = None, + bonus: Optional[Union[int, float]] = None, skip_unit_review: bool = False, ) -> None: """Approve the work done on this specific Unit""" @@ -139,7 +140,7 @@ def approve_work( worker_id=unit.worker_id, status=AgentState.STATUS_APPROVED, review_note=review_note, - bonus=bonus, + bonus=str(bonus), ) def soft_reject_work(self, review_note: Optional[str] = None) -> None: diff --git a/mephisto/abstractions/providers/prolific/prolific_utils.py b/mephisto/abstractions/providers/prolific/prolific_utils.py index 1d85cfa6c..0b9593a58 100644 --- a/mephisto/abstractions/providers/prolific/prolific_utils.py +++ b/mephisto/abstractions/providers/prolific/prolific_utils.py @@ -28,6 +28,7 @@ from .api.base_api_resource import CREDENTIALS_CONFIG_PATH from .api.client import ProlificClient from .api.data_models import BonusPayments +from .api.data_models import Message from .api.data_models import Participant from .api.data_models import ParticipantGroup from .api.data_models import Project @@ -596,7 +597,7 @@ def pay_bonus( client: ProlificClient, task_run_config: "DictConfig", worker_id: str, - bonus_amount: int, # in cents + bonus_amount: Union[int, float], # in cents study_id: str, *args, **kwargs, @@ -798,3 +799,12 @@ def reject_work( ) return None + + +def send_message(client: ProlificClient, study_id: str, participant_id: str, text: str) -> Message: + try: + message: Message = client.Messages.send(study_id, participant_id, text) + except (ProlificException, ValidationError): + logger.exception(f'Could not send message to participant "{participant_id}"') + raise + return message diff --git a/mephisto/abstractions/providers/prolific/prolific_worker.py b/mephisto/abstractions/providers/prolific/prolific_worker.py index d0e88a8f9..20c55b58f 100644 --- a/mephisto/abstractions/providers/prolific/prolific_worker.py +++ b/mephisto/abstractions/providers/prolific/prolific_worker.py @@ -13,8 +13,6 @@ from typing import Tuple from typing import TYPE_CHECKING -from omegaconf import DictConfig - from mephisto.abstractions.providers.prolific import prolific_utils from mephisto.abstractions.providers.prolific.api.client import ProlificClient from mephisto.abstractions.providers.prolific.provider_type import PROVIDER_TYPE @@ -298,6 +296,28 @@ def revoke_crowd_qualification(self, qualification_name: str) -> None: return None + def send_feedback_message(self, text: str, unit: "Unit") -> bool: + """Send feedback message to a worker""" + requester = cast( + "ProlificRequester", + self.db.find_requesters(provider_type=self.provider_type)[-1], + ) + + assert isinstance(requester, ProlificRequester), "Must be an Prolific requester" + + client = self._get_client(requester.requester_name) + datastore_unit = self.datastore.get_unit(unit.db_id) + prolific_study_id = datastore_unit["prolific_study_id"] + + prolific_utils.send_message( + client=client, + study_id=prolific_study_id, + participant_id=self.get_prolific_participant_id(), + text=text, + ) + + return True + @staticmethod def new(db: "MephistoDB", worker_id: str) -> "Worker": new_worker = ProlificWorker._register_worker(db, worker_id, PROVIDER_TYPE) diff --git a/mephisto/client/review_app/README.md b/mephisto/client/review_app/README.md index bd6d2e4d1..aefcc9c9a 100644 --- a/mephisto/client/review_app/README.md +++ b/mephisto/client/review_app/README.md @@ -63,11 +63,11 @@ _The "review" version will be shown to reviewer inside an iframe, while "regular 1. Create `review_app` subdirectory of your app, next to the main `app` subdirectory. Its content should include: - Main JS file `review.js` for ReviewApp - - Example: [review.js](../../../examples/remote_procedure/mnist/webapp/src/review.js) + - Example: [review.js](/examples/remote_procedure/mnist/webapp/src/review.js) - ReviewApp `reviewapp.jsx` - - Example: [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) - Separate Webpack config `webpack.config.review.js` - - Example: [webpack.config.review.js](../../../examples/remote_procedure/mnist/webapp/webpack.config.review.js) + - Example: [webpack.config.review.js](/examples/remote_procedure/mnist/webapp/webpack.config.review.js) 2. Specify in your Hydra YAML, under mephisto.blueprint section: ```json @@ -78,18 +78,18 @@ task_source_review: ${task_dir}/webapp/build/bundle.review.js ```json "build:review": "webpack --config=webpack.config.review.js --mode development" ``` -Example: [package.json](../../../examples/remote_procedure/mnist/webapp/package.json) +Example: [package.json](/examples/remote_procedure/mnist/webapp/package.json) 4. Build this "review" bundle by running `npm run build:review` from directory with `package.json`. 5. This `reviewapp.jsx` must satisfy 3 requirements, to interface with TaskReview: - Render "review" task version only upon receiving messages with Task data: - - Example: comment #1 in [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: comment #1 in [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) - Send messages with displayed "review" page height (to resize containing iframe and avoid scrollbars) - - Example: comment #2 in [reviewapp.jsx](../../../examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) + - Example: comment #2 in [reviewapp.jsx](/examples/remote_procedure/mnist/webapp/src/reviewapp.jsx) `