From 2f0b70f2834077b6e01c8f74eed21a8a2d471609 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Mon, 4 Apr 2022 17:00:17 -0400 Subject: [PATCH 1/2] Adding reasonable defaults to our examples for max concurrent units --- examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml | 3 +++ .../mnist/hydra_configs/conf/launch_with_local.yaml | 2 ++ .../template/hydra_configs/conf/launch_with_local.yaml | 3 +++ examples/simple_static_task/hydra_configs/conf/example.yaml | 2 ++ .../hydra_configs/conf/onboarding_example.yaml | 2 ++ examples/static_react_task/hydra_configs/conf/example.yaml | 2 ++ .../hydra_configs/conf/onboarding_example.yaml | 2 ++ 7 files changed, 16 insertions(+) diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml index 08b9086f9..d36f24f9f 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml @@ -8,3 +8,6 @@ mephisto: world_file: ${task_dir}/demo_worlds.py task_description_file: ${task_dir}/task_description.html num_conversations: 1 + task: + # We expect to be able to handle 25 concurrent conversations without issue + max_num_concurrent_units: 50 diff --git a/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml b/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml index f0dc89325..66ae5d557 100644 --- a/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml +++ b/examples/remote_procedure/mnist/hydra_configs/conf/launch_with_local.yaml @@ -20,3 +20,5 @@ mephisto: task_reward: 0.05 # NOTE will want real tags task_tags: "mnist,drawing,models,correction" + # We expect to handle 25 people using the MNIST model at once + max_num_concurrent_units: 25 diff --git a/examples/remote_procedure/template/hydra_configs/conf/launch_with_local.yaml b/examples/remote_procedure/template/hydra_configs/conf/launch_with_local.yaml index 199e8c37d..19a0442dd 100644 --- a/examples/remote_procedure/template/hydra_configs/conf/launch_with_local.yaml +++ b/examples/remote_procedure/template/hydra_configs/conf/launch_with_local.yaml @@ -20,3 +20,6 @@ mephisto: task_reward: 0.05 # NOTE will want real tags task_tags: "test,task,fix-me" + # NOTE Model-in-the-loop tasks need to be careful to configure only as many concurrent + # connections as their model can handle at once + max_num_concurrent_units: 40 diff --git a/examples/simple_static_task/hydra_configs/conf/example.yaml b/examples/simple_static_task/hydra_configs/conf/example.yaml index 198a1a495..51434b13e 100644 --- a/examples/simple_static_task/hydra_configs/conf/example.yaml +++ b/examples/simple_static_task/hydra_configs/conf/example.yaml @@ -16,3 +16,5 @@ mephisto: task_description: "This is a simple test of static tasks." task_reward: 0.3 task_tags: "static,task,testing" + # We expect to be able to handle 300 concurrent tasks without issue + max_num_concurrent_units: 300 diff --git a/examples/simple_static_task/hydra_configs/conf/onboarding_example.yaml b/examples/simple_static_task/hydra_configs/conf/onboarding_example.yaml index 6ca8b0861..554a84350 100644 --- a/examples/simple_static_task/hydra_configs/conf/onboarding_example.yaml +++ b/examples/simple_static_task/hydra_configs/conf/onboarding_example.yaml @@ -18,3 +18,5 @@ mephisto: task_description: "This is a simple test of static tasks." task_reward: 0.3 task_tags: "static,task,testing" + # We expect to be able to handle 300 concurrent tasks without issue + max_num_concurrent_units: 300 diff --git a/examples/static_react_task/hydra_configs/conf/example.yaml b/examples/static_react_task/hydra_configs/conf/example.yaml index 87b3288b0..d142a52c0 100644 --- a/examples/static_react_task/hydra_configs/conf/example.yaml +++ b/examples/static_react_task/hydra_configs/conf/example.yaml @@ -15,3 +15,5 @@ mephisto: task_description: "In this task, you'll be given a sentence. It is your job to rate it as either good or bad." task_reward: 0.05 task_tags: "test,simple,button" + # We expect to be able to handle 300 concurrent tasks without issue + max_num_concurrent_units: 300 diff --git a/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml b/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml index 72d37e1f3..e2c4fc9d1 100644 --- a/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml +++ b/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml @@ -15,3 +15,5 @@ mephisto: task_description: "In this task, you'll be given a sentence. It is your job to rate it as either good or bad." task_reward: 0.05 task_tags: "test,simple,button" + # We expect to be able to handle 300 concurrent tasks without issue + max_num_concurrent_units: 300 From b1a623558f596500a9e69bab4830361b624ba688 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Thu, 7 Apr 2022 10:51:55 -0400 Subject: [PATCH 2/2] Conversation clarification --- examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml index d36f24f9f..c51fd7ebb 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml @@ -10,4 +10,4 @@ mephisto: num_conversations: 1 task: # We expect to be able to handle 25 concurrent conversations without issue - max_num_concurrent_units: 50 + max_num_concurrent_units: 50 # 25 convos * 2 people per