From 04efb9d52ef5ad5c09c8b654d3edf72af5d2923b Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Wed, 6 Dec 2023 06:05:30 -0800 Subject: [PATCH 1/6] Remove duplicate `boto3` requirement (#1875) * Remove duplicate `boto3` requirement There is a boto3 requirement in requirements.txt and in tests/requirements.txt. This will eventually lead to some confusion because it's prone to get out of sync. It is a core dependency, so I kept it in requirements.txt. * Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ tests/requirements.txt | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b07900e..053795f92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Operations + +- Remove `boto3` dependency from `tests/requirements.txt` + ## [0.232.0-rc.0] - 2023-12-01 ### Authors diff --git a/tests/requirements.txt b/tests/requirements.txt index 4db410654..99f936c73 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,3 @@ -boto3>=1.26.110 detect-secrets>=1.3.0 flake8>=5.0.4 httpx>=0.24.1 From c78a1d8dab44dc7644a96aa4466429a428417757 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Wed, 6 Dec 2023 23:11:06 -0800 Subject: [PATCH 2/6] Allow for `cloudpickle>=3.0.0` (#1873) * [Not for merge] Testing behavior of `cloudpickle` 3.0.0 * Update CHANGELOG.md --- CHANGELOG.md | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 053795f92..afd38c20f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Operations +- Allow `cloudpickle` >= 3.0.0 - Remove `boto3` dependency from `tests/requirements.txt` ## [0.232.0-rc.0] - 2023-12-01 diff --git a/requirements.txt b/requirements.txt index bdffc7147..886884819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiohttp>=3.8.1 alembic>=1.8.0 boto3>=1.26.110 click>=8.1.3 -cloudpickle>=2.0.0,<3 +cloudpickle>=2.0.0 dask[distributed]>=2022.6.0 fastapi>=0.100.0 filelock>=3.12.2 From a6a6999988ee8a8daecc486eb1433b5cc2dd19d0 Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Mon, 11 Dec 2023 09:32:26 -0500 Subject: [PATCH 3/6] Reduce the number of assets to upload during dispatch (#1879) Initialize unset assets to `None` and don' upload them. When downloading assets during `get_result()`, ignore assets with zero size. Also require asset sizes when retrieving asset updates in `Executor.receive()`. --- CHANGELOG.md | 11 +++ covalent/_dispatcher_plugins/local.py | 8 +- covalent/_results_manager/result.py | 4 +- covalent/_results_manager/results_manager.py | 27 ++++-- covalent/_serialize/common.py | 3 + covalent/_serialize/electron.py | 17 ++-- covalent/_serialize/lattice.py | 4 +- covalent/_shared_files/schemas/asset.py | 6 +- covalent/executor/executor_plugins/dask.py | 20 ++++- covalent/executor/executor_plugins/local.py | 17 +--- covalent/executor/utils/wrappers.py | 75 +++++++++++++---- .../_core/data_modules/asset_manager.py | 82 +++++++++++-------- covalent_dispatcher/_dal/asset.py | 21 +++-- covalent_dispatcher/_dal/base.py | 32 +++----- covalent_dispatcher/_db/upsert.py | 6 +- covalent_dispatcher/_object_store/base.py | 3 + covalent_dispatcher/_object_store/local.py | 10 ++- covalent_dispatcher/_service/assets.py | 5 +- .../v1/routes/end_points/electron_routes.py | 4 + .../asset_manager_db_integration_test.py | 63 ++++++++++++-- .../_core/runner_ng_test.py | 3 + .../_dal/asset_test.py | 12 +-- .../_dal/electron_test.py | 2 +- .../_dal/importers/result_import_test.py | 4 +- .../_db/update_test.py | 11 +-- .../_db/upsert_test.py | 2 +- .../_object_store/local_test.py | 4 +- .../dispatcher_plugins/local_test.py | 6 +- .../executor/executor_plugins/dask_test.py | 17 ++++ tests/functional_tests/local_executor_test.py | 2 +- tests/functional_tests/workflow_stack_test.py | 4 +- 31 files changed, 333 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afd38c20f..d7ffb8aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Changed + +- SDK no longer uploads empty assets when submitting a dispatch. +- Results Manager avoids downloading assets with size 0. +- Local and Dask executor plugins now return accurate sizes of task + artifacts. +- Size (number of bytes) is now a required attribute whenever updating + asset metadata. Although the exact numerical value is not yet + important, whether the size is reported to be zero or positive does + have consequences. + ### Operations - Allow `cloudpickle` >= 3.0.0 diff --git a/covalent/_dispatcher_plugins/local.py b/covalent/_dispatcher_plugins/local.py index f0df0f8a0..8760cec96 100644 --- a/covalent/_dispatcher_plugins/local.py +++ b/covalent/_dispatcher_plugins/local.py @@ -593,15 +593,19 @@ def upload_assets(manifest: ResultSchema): def _upload(assets: List[AssetSchema]): local_scheme_prefix = "file://" total = len(assets) + number_uploaded = 0 for i, asset in enumerate(assets): - if not asset.remote_uri: + if not asset.remote_uri or not asset.uri: app_log.debug(f"Skipping asset {i+1} out of {total}") continue if asset.remote_uri.startswith(local_scheme_prefix): copy_file_locally(asset.uri, asset.remote_uri) + number_uploaded += 1 else: _upload_asset(asset.uri, asset.remote_uri) - app_log.debug(f"uploaded {i+1} out of {total} assets.") + number_uploaded += 1 + app_log.debug(f"Uploaded asset {i+1} out of {total}.") + app_log.debug(f"uploaded {number_uploaded} assets.") def _upload_asset(local_uri, remote_uri): diff --git a/covalent/_results_manager/result.py b/covalent/_results_manager/result.py index c9d42c2fd..a42f514a6 100644 --- a/covalent/_results_manager/result.py +++ b/covalent/_results_manager/result.py @@ -90,11 +90,11 @@ def __init__(self, lattice: Lattice, dispatch_id: str = "") -> None: self._task_failed = False self._task_cancelled = False - self._result = TransportableObject(None) + self._result = None self._num_nodes = -1 - self._error = "" + self._error = None def __str__(self): """String representation of the result object""" diff --git a/covalent/_results_manager/results_manager.py b/covalent/_results_manager/results_manager.py index 458d746d4..9372a43e0 100644 --- a/covalent/_results_manager/results_manager.py +++ b/covalent/_results_manager/results_manager.py @@ -248,26 +248,35 @@ def download_asset(remote_uri: str, local_path: str, chunk_size: int = 1024 * 10 def _download_result_asset(manifest: dict, results_dir: str, key: str): remote_uri = manifest["assets"][key]["remote_uri"] - local_path = get_result_asset_path(results_dir, key) - download_asset(remote_uri, local_path) - manifest["assets"][key]["uri"] = f"file://{local_path}" + size = manifest["assets"][key]["size"] + + if size > 0: + local_path = get_result_asset_path(results_dir, key) + download_asset(remote_uri, local_path) + manifest["assets"][key]["uri"] = f"file://{local_path}" def _download_lattice_asset(manifest: dict, results_dir: str, key: str): lattice_assets = manifest["lattice"]["assets"] remote_uri = lattice_assets[key]["remote_uri"] - local_path = get_lattice_asset_path(results_dir, key) - download_asset(remote_uri, local_path) - lattice_assets[key]["uri"] = f"file://{local_path}" + size = lattice_assets[key]["size"] + + if size > 0: + local_path = get_lattice_asset_path(results_dir, key) + download_asset(remote_uri, local_path) + lattice_assets[key]["uri"] = f"file://{local_path}" def _download_node_asset(manifest: dict, results_dir: str, node_id: int, key: str): node = manifest["lattice"]["transport_graph"]["nodes"][node_id] node_assets = node["assets"] remote_uri = node_assets[key]["remote_uri"] - local_path = get_node_asset_path(results_dir, node_id, key) - download_asset(remote_uri, local_path) - node_assets[key]["uri"] = f"file://{local_path}" + size = node_assets[key]["size"] + + if size > 0: + local_path = get_node_asset_path(results_dir, node_id, key) + download_asset(remote_uri, local_path) + node_assets[key]["uri"] = f"file://{local_path}" def _load_result_asset(manifest: dict, key: str): diff --git a/covalent/_serialize/common.py b/covalent/_serialize/common.py index c36f7d685..341e112f0 100644 --- a/covalent/_serialize/common.py +++ b/covalent/_serialize/common.py @@ -136,6 +136,9 @@ def save_asset(data: Any, data_type: AssetType, storage_path: str, filename: str scheme = "file" + if data is None: + return AssetSchema(size=0) + serialized = serialize_asset(data, data_type) digest = _sha1_asset(serialized) path = Path(storage_path) / filename diff --git a/covalent/_serialize/electron.py b/covalent/_serialize/electron.py index 3229875ae..03a3b1ae2 100644 --- a/covalent/_serialize/electron.py +++ b/covalent/_serialize/electron.py @@ -26,7 +26,6 @@ ElectronSchema, ) from .._shared_files.util_classes import RESULT_STATUS, Status -from .._workflow.transportable_object import TransportableObject from .common import AssetType, load_asset, save_asset __all__ = [ @@ -105,7 +104,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["function"], ) - function_string = node_attrs.get("function_string", "") + function_string = node_attrs.get("function_string", None) function_string_asset = save_asset( function_string, ASSET_TYPES["function_string"], @@ -113,7 +112,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["function_string"], ) - node_value = node_attrs.get("value", TransportableObject(None)) + node_value = node_attrs.get("value", None) value_asset = save_asset( node_value, ASSET_TYPES["value"], @@ -121,7 +120,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["value"], ) - node_output = node_attrs.get("output", TransportableObject(None)) + node_output = node_attrs.get("output", None) output_asset = save_asset( node_output, ASSET_TYPES["output"], @@ -129,7 +128,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["output"], ) - node_stdout = node_attrs.get("stdout", "") + node_stdout = node_attrs.get("stdout", None) stdout_asset = save_asset( node_stdout, ASSET_TYPES["stdout"], @@ -137,7 +136,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["stdout"], ) - node_stderr = node_attrs.get("stderr", "") + node_stderr = node_attrs.get("stderr", None) stderr_asset = save_asset( node_stderr, ASSET_TYPES["stderr"], @@ -145,7 +144,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["stderr"], ) - qelectron_db = node_attrs.get("qelectron_db", bytes()) + qelectron_db = node_attrs.get("qelectron_db", None) qelectron_db_asset = save_asset( qelectron_db, ASSET_TYPES["qelectron_db"], @@ -153,7 +152,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["qelectron_db"], ) - node_error = node_attrs.get("error", "") + node_error = node_attrs.get("error", None) error_asset = save_asset( node_error, ASSET_TYPES["error"], @@ -230,7 +229,7 @@ def _deserialize_node_assets(ea: ElectronAssets) -> dict: def _get_node_custom_assets(node_attrs: dict) -> Dict[str, AssetSchema]: if "custom_asset_keys" in node_attrs["metadata"]: - return {key: AssetSchema() for key in node_attrs["metadata"]["custom_asset_keys"]} + return {key: AssetSchema(size=0) for key in node_attrs["metadata"]["custom_asset_keys"]} def serialize_node(node_id: int, node_attrs: dict, node_storage_path) -> ElectronSchema: diff --git a/covalent/_serialize/lattice.py b/covalent/_serialize/lattice.py index ebe405ac5..b188b8edf 100644 --- a/covalent/_serialize/lattice.py +++ b/covalent/_serialize/lattice.py @@ -102,7 +102,7 @@ def _serialize_lattice_assets(lat, storage_path: str) -> LatticeAssets: ASSET_FILENAME_MAP["workflow_function_string"], ) - docstring = "" if lat.__doc__ is None else lat.__doc__ + docstring = lat.__doc__ docstring_asset = save_asset( docstring, ASSET_TYPES["doc"], @@ -208,7 +208,7 @@ def _deserialize_lattice_assets(assets: LatticeAssets) -> dict: def _get_lattice_custom_assets(lat: Lattice) -> Dict[str, AssetSchema]: if "custom_asset_keys" in lat.metadata: - return {key: AssetSchema() for key in lat.metadata["custom_asset_keys"]} + return {key: AssetSchema(size=0) for key in lat.metadata["custom_asset_keys"]} def serialize_lattice(lat, storage_path: str) -> LatticeSchema: diff --git a/covalent/_shared_files/schemas/asset.py b/covalent/_shared_files/schemas/asset.py index 957560cc6..ff93d92bc 100644 --- a/covalent/_shared_files/schemas/asset.py +++ b/covalent/_shared_files/schemas/asset.py @@ -28,11 +28,13 @@ class AssetSchema(BaseModel): remote_uri: Optional[str] = None # Size of the asset in bytes - size: Optional[int] = 0 + size: int class AssetUpdate(BaseModel): remote_uri: Optional[str] = None - size: Optional[int] = None digest_alg: Optional[str] = None digest: Optional[str] = None + + # Size of the asset in bytes + size: int diff --git a/covalent/executor/executor_plugins/dask.py b/covalent/executor/executor_plugins/dask.py index 73b719b90..fa7c8c315 100644 --- a/covalent/executor/executor_plugins/dask.py +++ b/covalent/executor/executor_plugins/dask.py @@ -307,19 +307,27 @@ async def receive(self, task_group_metadata: Dict, data: Any) -> List[TaskUpdate if terminal_status == RESULT_STATUS.CANCELLED: output_uri = "" + output_size = 0 stdout_uri = "" + stdout_size = 0 stderr_uri = "" + stderr_size = 0 qelectron_db_uri = "" + qelectron_db_size = 0 else: result_path = os.path.join(self.cache_dir, f"result-{dispatch_id}:{task_id}.json") with open(result_path, "r") as f: result_summary = json.load(f) node_id = result_summary["node_id"] - output_uri = result_summary["output_uri"] - stdout_uri = result_summary["stdout_uri"] - stderr_uri = result_summary["stderr_uri"] - qelectron_db_uri = result_summary["qelectron_db_uri"] + output_uri = result_summary["output"]["uri"] + output_size = result_summary["output"]["size"] + stdout_uri = result_summary["stdout"]["uri"] + stdout_size = result_summary["stdout"]["size"] + stderr_uri = result_summary["stderr"]["uri"] + stderr_size = result_summary["stderr"]["size"] + qelectron_db_uri = result_summary["qelectron_db"]["uri"] + qelectron_db_size = result_summary["qelectron_db"]["size"] exception_raised = result_summary["exception_occurred"] terminal_status = ( @@ -333,15 +341,19 @@ async def receive(self, task_group_metadata: Dict, data: Any) -> List[TaskUpdate "assets": { "output": { "remote_uri": output_uri, + "size": output_size, }, "stdout": { "remote_uri": stdout_uri, + "size": stdout_size, }, "stderr": { "remote_uri": stderr_uri, + "size": stderr_size, }, "qelectron_db": { "remote_uri": qelectron_db_uri, + "size": qelectron_db_size, }, }, } diff --git a/covalent/executor/executor_plugins/local.py b/covalent/executor/executor_plugins/local.py index 5cf2659b9..38047d040 100644 --- a/covalent/executor/executor_plugins/local.py +++ b/covalent/executor/executor_plugins/local.py @@ -216,24 +216,13 @@ def _receive(self, task_group_metadata: Dict, data: Any) -> List[TaskUpdate]: received = ReceiveModel.model_validate(data) terminal_status = Status(received.status.value) + # Don't update any asset metadata since all assets will be + # pushed from the executor task_result = { "dispatch_id": dispatch_id, "node_id": task_id, "status": terminal_status, - "assets": { - "output": { - "remote_uri": "", - }, - "stdout": { - "remote_uri": "", - }, - "stderr": { - "remote_uri": "", - }, - "qelectron_db": { - "remote_uri": "", - }, - }, + "assets": {}, } task_results.append(TaskUpdate(**task_result)) diff --git a/covalent/executor/utils/wrappers.py b/covalent/executor/utils/wrappers.py index 15f406b51..90dec9e64 100644 --- a/covalent/executor/utils/wrappers.py +++ b/covalent/executor/utils/wrappers.py @@ -469,12 +469,30 @@ def run_task_from_uris_alt( resources["inputs"][task_id] = result_uri + output_size = len(ser_output) + qelectron_db_size = len(qelectron_db_bytes) + stdout.flush() + stderr.flush() + stdout_size = os.path.getsize(stdout_uri) + stderr_size = os.path.getsize(stderr_uri) result_summary = { "node_id": task_id, - "output_uri": result_uri, - "stdout_uri": stdout_uri, - "stderr_uri": stderr_uri, - "qelectron_db_uri": qelectron_db_uri, + "output": { + "uri": result_uri, + "size": output_size, + }, + "stdout": { + "uri": stdout_uri, + "size": stdout_size, + }, + "stderr": { + "uri": stderr_uri, + "size": stderr_size, + }, + "qelectron_db": { + "uri": qelectron_db_uri, + "size": qelectron_db_size, + }, "exception_occurred": exception_occurred, } @@ -482,13 +500,30 @@ def run_task_from_uris_alt( exception_occurred = True tb = "".join(traceback.TracebackException.from_exception(ex).format()) print(tb, file=sys.stderr) - result_uri = None + result_uri = "" + stdout.flush() + stderr.flush() + stdout_size = os.path.getsize(stdout_uri) + stderr_size = os.path.getsize(stderr_uri) + qelectron_db_size = len(qelectron_db_bytes) result_summary = { "node_id": task_id, - "output_uri": result_uri, - "stdout_uri": stdout_uri, - "stderr_uri": stderr_uri, - "qelectron_db_uri": qelectron_db_uri, + "output": { + "uri": result_uri, + "size": 0, + }, + "stdout": { + "uri": stdout_uri, + "size": stdout_size, + }, + "stderr": { + "uri": stderr_uri, + "size": stderr_size, + }, + "qelectron_db": { + "uri": qelectron_db_uri, + "size": qelectron_db_size, + }, "exception_occurred": exception_occurred, } @@ -508,11 +543,23 @@ def run_task_from_uris_alt( if n < len(task_ids): for i in range(n, len(task_ids)): result_summary = { - "node_id": task_ids[i], - "output_uri": "", - "stdout_uri": "", - "stderr_uri": "", - "qelectron_db_uri": "", + "node_id": task_id, + "output": { + "uri": "", + "size": 0, + }, + "stdout": { + "uri": "", + "size": 0, + }, + "stderr": { + "uri": "", + "size": 0, + }, + "qelectron_db": { + "uri": "", + "size": 0, + }, "exception_occurred": True, } diff --git a/covalent_dispatcher/_core/data_modules/asset_manager.py b/covalent_dispatcher/_core/data_modules/asset_manager.py index 984ea6e46..3bd0a6ff3 100644 --- a/covalent_dispatcher/_core/data_modules/asset_manager.py +++ b/covalent_dispatcher/_core/data_modules/asset_manager.py @@ -26,7 +26,6 @@ from covalent._shared_files.schemas.asset import AssetUpdate from ..._dal.result import get_result_object as get_result_object -from .utils import run_in_executor app_log = logger.app_log am_pool = ThreadPoolExecutor() @@ -36,18 +35,32 @@ async def upload_asset_for_nodes(dispatch_id: str, key: str, dest_uris: dict): """Typical keys: "output", "deps", "call_before", "call_after", "function""" + loop = asyncio.get_running_loop() + await loop.run_in_executor( + am_pool, + upload_asset_for_nodes_sync, + dispatch_id, + key, + dest_uris, + ) + + +def upload_asset_for_nodes_sync(dispatch_id: str, key: str, dest_uris: dict): result_object = get_result_object(dispatch_id, bare=True) tg = result_object.lattice.transport_graph - loop = asyncio.get_running_loop() - futs = [] - for node_id, dest_uri in dest_uris.items(): - if dest_uri: - node = tg.get_node(node_id) - asset = node.get_asset(key, session=None) - futs.append(loop.run_in_executor(am_pool, asset.upload, dest_uri)) + uploads_by_node = {} + + with result_object.session() as session: + for node_id, dest_uri in dest_uris.items(): + if dest_uri: + node = tg.get_node(node_id, session) + asset = node.get_asset(key, session) + uploads_by_node[node_id] = (asset, dest_uri) - await asyncio.gather(*futs) + for _, pending_upload in uploads_by_node.items(): + asset, dest_uri = pending_upload + asset.upload(dest_uri) async def download_assets_for_node( @@ -55,35 +68,40 @@ async def download_assets_for_node( ): # Keys for src_uris: "output", "stdout", "stderr" + loop = asyncio.get_running_loop() + await loop.run_in_executor( + am_pool, + download_assets_for_node_sync, + dispatch_id, + node_id, + asset_updates, + ) + + +def download_assets_for_node_sync( + dispatch_id: str, node_id: int, asset_updates: Dict[str, AssetUpdate] +): result_object = get_result_object(dispatch_id, bare=True) tg = result_object.lattice.transport_graph node = tg.get_node(node_id) - loop = asyncio.get_running_loop() - futs = [] db_updates = {} - # Mapping from asset key to (non-empty) remote uri + # Mapping from asset key to (Asset, remote uri) assets_to_download = {} # Prepare asset metadata update; prune empty fields - for key in asset_updates: - update = {} - asset = asset_updates[key].dict() - if asset["remote_uri"]: - assets_to_download[key] = asset["remote_uri"] - # Prune empty fields - for attr, val in asset.items(): - if val is not None: - update[attr] = val - if update: - db_updates[key] = update - - # Update metadata using the designated DB worker thread - await run_in_executor(node.update_assets, db_updates) - - for key, remote_uri in assets_to_download.items(): - asset = node.get_asset(key, session=None) - # Download assets concurrently. - futs.append(loop.run_in_executor(am_pool, asset.download, remote_uri)) - await asyncio.gather(*futs) + with result_object.session() as session: + for key in asset_updates: + asset_update = asset_updates[key] + if asset_update.remote_uri: + asset = node.get_asset(key, session) + assets_to_download[key] = (asset, asset_update.remote_uri) + # Prune unset fields + db_updates[key] = {attr: val for attr, val in asset_update if val is not None} + + node.update_assets(db_updates, session) + + for key, pending_download in assets_to_download.items(): + asset, remote_uri = pending_download + asset.download(remote_uri) diff --git a/covalent_dispatcher/_dal/asset.py b/covalent_dispatcher/_dal/asset.py index 8eda3a740..d524dc389 100644 --- a/covalent_dispatcher/_dal/asset.py +++ b/covalent_dispatcher/_dal/asset.py @@ -109,8 +109,16 @@ def size(self) -> int: def set_remote(self, session: Session, uri: str): self.update(session, values={"remote_uri": uri}) - def store_data(self, data: Any) -> None: - self.object_store.store_file(self.storage_path, self.object_key, data) + def store_data(self, data: Any, session: Session) -> None: + digest, size = self.object_store.store_file(self.storage_path, self.object_key, data) + self.update( + session, + values={ + "digest_alg": digest.algorithm, + "digest": digest.hexdigest, + "size": size, + }, + ) def load_data(self) -> Any: return self.object_store.load_file(self.storage_path, self.object_key) @@ -146,9 +154,12 @@ def copy_asset(src: Asset, dest: Asset): dest The destination asset """ - scheme = dest.storage_type.value - dest_uri = scheme + "://" + os.path.join(dest.storage_path, dest.object_key) - src.upload(dest_uri) + if src.size > 0: + scheme = dest.storage_type.value + dest_uri = scheme + "://" + os.path.join(dest.storage_path, dest.object_key) + src.upload(dest_uri) + else: + app_log.debug(f"Refusing to copy zero-sized asset {src.internal_uri}") def copy_asset_meta(session: Session, src: Asset, dest: Asset): diff --git a/covalent_dispatcher/_dal/base.py b/covalent_dispatcher/_dal/base.py index 1ecb84fcd..071d9a44b 100644 --- a/covalent_dispatcher/_dal/base.py +++ b/covalent_dispatcher/_dal/base.py @@ -126,15 +126,13 @@ def incr_metadata(self, key: str, delta: int, session: Session): attr = type(self).meta_record_map(key) self.metadata.incr(session, increments={attr: delta}) - def get_asset(self, key: str, session: Session) -> Asset: - if key not in self.assets: - if session: - asset_id = self.get_asset_ids(session, [key])[key] - self.assets[key] = Asset.from_id(asset_id, session) - else: - with self.session() as session: - asset_id = self.get_asset_ids(session, [key])[key] - self.assets[key] = Asset.from_id(asset_id, session) + def get_asset(self, key: str, session: Session, refresh: bool = True) -> Asset: + if key in self.assets: + if refresh: + self.assets[key].refresh(session, fields=FIELDS) + else: + asset_id = self.get_asset_ids(session, [key])[key] + self.assets[key] = Asset.from_id(asset_id, session) return self.assets[key] @@ -144,17 +142,11 @@ def populate_asset_map(self, session: Session): for key, asset_id in asset_links.items(): self.assets[key] = Asset.from_id(asset_id, session) - def update_assets(self, updates: Dict[str, Dict], session: Session = None): + def update_assets(self, updates: Dict[str, Dict], session: Session): """Bulk update associated assets""" - if session: - for key, values in updates.items(): - asset = self.get_asset(key, session) - asset.update(session, values=values) - else: - with self.session() as session: - for key, values in updates.items(): - asset = self.get_asset(key, session) - asset.update(session, values=values) + for key, values in updates.items(): + asset = self.get_asset(key, session) + asset.update(session, values=values) def _get_value(self, key: str, session: Session, refresh: bool = True) -> Any: if key in self.computed_fields: @@ -176,7 +168,7 @@ def _set_value(self, key: str, val: Any, session: Session) -> None: if key in type(self).metadata_keys: self.set_metadata(key, val, session) else: - self.get_asset(key, session).store_data(val) + self.get_asset(key, session).store_data(val, session) def set_value(self, key: str, val: Any, session: Session = None) -> None: if session is not None: diff --git a/covalent_dispatcher/_db/upsert.py b/covalent_dispatcher/_db/upsert.py index 64bcf2375..cc2e76309 100644 --- a/covalent_dispatcher/_db/upsert.py +++ b/covalent_dispatcher/_db/upsert.py @@ -121,13 +121,14 @@ def _lattice_data(session: Session, result: Result, electron_id: int = None) -> ("cova_imports", LATTICE_COVA_IMPORTS_FILENAME, result.lattice.cova_imports), ("lattice_imports", LATTICE_LATTICE_IMPORTS_FILENAME, result.lattice.lattice_imports), ]: - digest = local_store.store_file(data_storage_path, filename, data) + digest, size = local_store.store_file(data_storage_path, filename, data) asset_record_kwargs = { "storage_type": LATTICE_STORAGE_TYPE, "storage_path": str(data_storage_path), "object_key": filename, "digest_alg": digest.algorithm, "digest": digest.hexdigest, + "size": size, } assets[key] = Asset.create(session, insert_kwargs=asset_record_kwargs, flush=True) @@ -310,13 +311,14 @@ def _electron_data( ("error", ELECTRON_ERROR_FILENAME, node_error), ("output", ELECTRON_RESULTS_FILENAME, node_output), ]: - digest = local_store.store_file(node_path, filename, data) + digest, size = local_store.store_file(node_path, filename, data) asset_record_kwargs = { "storage_type": ELECTRON_STORAGE_TYPE, "storage_path": str(node_path), "object_key": filename, "digest_alg": digest.algorithm, "digest": digest.hexdigest, + "size": size, } assets[key] = Asset.create(session, insert_kwargs=asset_record_kwargs, flush=True) diff --git a/covalent_dispatcher/_object_store/base.py b/covalent_dispatcher/_object_store/base.py index 09bfad137..7329c6671 100644 --- a/covalent_dispatcher/_object_store/base.py +++ b/covalent_dispatcher/_object_store/base.py @@ -35,6 +35,9 @@ def scheme(cls) -> str: def digest(self, bucket_name: str, object_key: str) -> Digest: raise NotImplementedError + def size(self, bucket_name: str, object_key: str) -> int: + raise NotImplementedError + def get_uri_components( self, dispatch_id: str, node_id: Optional[int], asset_key: str ) -> Tuple[str, str]: diff --git a/covalent_dispatcher/_object_store/local.py b/covalent_dispatcher/_object_store/local.py index 5e8d08616..6bd4e7ec9 100644 --- a/covalent_dispatcher/_object_store/local.py +++ b/covalent_dispatcher/_object_store/local.py @@ -69,7 +69,7 @@ def size(self, bucket_name: str, object_key: str) -> int: path = os.path.join(bucket_name, object_key) try: - return os.path.size(path) + return os.path.getsize(path) except OSError: return 0 @@ -104,9 +104,12 @@ def get_uri_components( return storage_path, object_key - def store_file(self, storage_path: str, filename: str, data: Any = None) -> Digest: + def store_file(self, storage_path: str, filename: str, data: Any = None) -> Tuple[Digest, int]: """This function writes data corresponding to the filepaths in the DB.""" + if data is None: + return Digest(algorithm="sha1", hexdigest=""), 0 + if filename.endswith(".pkl"): with open(Path(storage_path) / filename, "wb") as f: cloudpickle.dump(data, f) @@ -136,7 +139,8 @@ def store_file(self, storage_path: str, filename: str, data: Any = None) -> Dige raise InvalidFileExtension("The file extension is not supported.") digest = self.digest(bucket_name=storage_path, object_key=filename) - return digest + size = self.size(bucket_name=storage_path, object_key=filename) + return digest, size def load_file(self, storage_path: str, filename: str) -> Any: """This function loads data for the filenames in the DB.""" diff --git a/covalent_dispatcher/_service/assets.py b/covalent_dispatcher/_service/assets.py index 83caffd07..0664e5058 100644 --- a/covalent_dispatcher/_service/assets.py +++ b/covalent_dispatcher/_service/assets.py @@ -256,7 +256,7 @@ async def upload_node_asset( content_length: (header) digest: (header) """ - app_log.debug(f"Requested asset {key} for node {dispatch_id}:{node_id}") + app_log.debug(f"Uploading node asset {dispatch_id}:{node_id}:{key} ({content_length} bytes) ") try: metadata = {"size": content_length, "digest_alg": digest_alg, "digest": digest} @@ -294,6 +294,7 @@ async def upload_dispatch_asset( content_length: (header) digest: (header) """ + app_log.debug(f"Uploading dispatch asset {dispatch_id}:{key} ({content_length} bytes) ") try: metadata = {"size": content_length, "digest_alg": digest_alg, "digest": digest} internal_uri = await _run_in_executor( @@ -329,6 +330,7 @@ async def upload_lattice_asset( digest: (header) """ try: + app_log.debug(f"Uploading lattice asset {dispatch_id}:{key} ({content_length} bytes) ") metadata = {"size": content_length, "digest_alg": digest_alg, "digest": digest} internal_uri = await _run_in_executor( _update_lattice_asset_metadata, @@ -513,6 +515,7 @@ async def _transfer_data(req: Request, destination_url: str): # Stream data to a temporary file, then replace the destination # file atomically tmp_path = f"{dest_path}.tmp" + app_log.debug(f"Streaming file upload to {tmp_path}") async with aiofiles.open(tmp_path, "wb") as f: async for chunk in req.stream(): diff --git a/covalent_ui/api/v1/routes/end_points/electron_routes.py b/covalent_ui/api/v1/routes/end_points/electron_routes.py index 616f1c806..edd074dc5 100644 --- a/covalent_ui/api/v1/routes/end_points/electron_routes.py +++ b/covalent_ui/api/v1/routes/end_points/electron_routes.py @@ -235,6 +235,10 @@ def get_electron_file(dispatch_id: uuid.UUID, electron_id: int, name: ElectronFi # is only used for fatal dispatcher-executor interaction errors error_response = handler.read_from_text(result["error_filename"]) stderr_response = handler.read_from_text(result["stderr_filename"]) + if error_response is None: + error_response = "" + if stderr_response is None: + stderr_response = "" response = stderr_response + error_response return ElectronFileResponse(data=response) elif name == "qelectron_db": diff --git a/tests/covalent_dispatcher_tests/_core/data_modules/asset_manager_db_integration_test.py b/tests/covalent_dispatcher_tests/_core/data_modules/asset_manager_db_integration_test.py index efe75dddd..59c03a685 100644 --- a/tests/covalent_dispatcher_tests/_core/data_modules/asset_manager_db_integration_test.py +++ b/tests/covalent_dispatcher_tests/_core/data_modules/asset_manager_db_integration_test.py @@ -72,8 +72,7 @@ def get_mock_srvresult(sdkres, test_db) -> Result: return get_result_object(sdkres.dispatch_id) -@pytest.mark.asyncio -async def test_upload_asset_for_nodes(test_db, mocker): +def test_upload_asset_for_nodes(test_db, mocker): sdkres = get_mock_result() sdkres._initialize_nodes() @@ -95,7 +94,7 @@ async def test_upload_asset_for_nodes(test_db, mocker): dest_uri_0 = os.path.join("file://", dest_path_0) dest_uri_2 = os.path.join("file://", dest_path_2) - await am.upload_asset_for_nodes(srvres.dispatch_id, "stdout", {0: dest_uri_0, 2: dest_uri_2}) + am.upload_asset_for_nodes_sync(srvres.dispatch_id, "stdout", {0: dest_uri_0, 2: dest_uri_2}) with open(dest_path_0, "r") as f: assert f.read() == "Hello!\n" @@ -108,7 +107,22 @@ async def test_upload_asset_for_nodes(test_db, mocker): @pytest.mark.asyncio -async def test_download_assets_for_node(test_db, mocker): +async def test_async_upload(mocker): + mock_sync_upload = mocker.patch( + "covalent_dispatcher._core.data_modules.asset_manager.upload_asset_for_nodes_sync" + ) + dispatch_id = "dispatch_id" + asset_name = "stdout" + uris = {} + await am.upload_asset_for_nodes( + dispatch_id, + asset_name, + uris, + ) + mock_sync_upload.assert_called_with(dispatch_id, asset_name, uris) + + +def test_download_assets_for_node(test_db, mocker): sdkres = get_mock_result() sdkres._initialize_nodes() @@ -116,8 +130,6 @@ async def test_download_assets_for_node(test_db, mocker): mocker.patch("covalent_dispatcher._db.upsert.workflow_db", test_db) mocker.patch("covalent_dispatcher._dal.base.workflow_db", test_db) - mock_update_assets = mocker.patch("covalent_dispatcher._dal.electron.Electron.update_assets") - srvres = get_mock_srvresult(sdkres, test_db) with tempfile.NamedTemporaryFile("w", delete=False, suffix=".txt") as temp: @@ -134,10 +146,14 @@ async def test_download_assets_for_node(test_db, mocker): assets = { "output": { "remote_uri": "", + "digest": "", + "size": 0, }, - "stdout": {"remote_uri": src_uri_stdout, "size": None, "digest": "0af23"}, + "stdout": {"remote_uri": src_uri_stdout, "size": 5, "digest": "0af23"}, "stderr": { "remote_uri": src_uri_stderr, + "digest": "", + "size": 0, }, } assets = {k: AssetUpdate(**v) for k, v in assets.items()} @@ -145,21 +161,50 @@ async def test_download_assets_for_node(test_db, mocker): expected_update = { "output": { "remote_uri": "", + "digest": "", + "size": 0, }, "stdout": { "remote_uri": src_uri_stdout, "digest": "0af23", + "size": 5, }, "stderr": { "remote_uri": src_uri_stderr, + "digest": "", + "size": 0, }, } - await am.download_assets_for_node( + am.download_assets_for_node_sync( srvres.dispatch_id, 0, assets, ) - mock_update_assets.assert_called_with(expected_update) assert srvres.lattice.transport_graph.get_node_value(0, "stdout") == "Hello!\n" assert srvres.lattice.transport_graph.get_node_value(0, "stderr") == "Bye!\n" + + # CHeck metadata + with test_db.session() as session: + node = srvres.lattice.transport_graph.get_node(0, session) + for key, attrs in expected_update.items(): + asset = node.get_asset(key, session) + assert asset.size == attrs["size"] + assert asset.remote_uri == attrs["remote_uri"] + assert asset.digest == attrs["digest"] + + +@pytest.mark.asyncio +async def test_async_download(mocker): + mock_sync_download = mocker.patch( + "covalent_dispatcher._core.data_modules.asset_manager.download_assets_for_node_sync" + ) + dispatch_id = "mock_dispatch_id" + node_id = 0 + updates = {} + await am.download_assets_for_node( + dispatch_id, + node_id, + updates, + ) + mock_sync_download.asset_called_with(dispatch_id, node_id, updates) diff --git a/tests/covalent_dispatcher_tests/_core/runner_ng_test.py b/tests/covalent_dispatcher_tests/_core/runner_ng_test.py index fd88b0839..c1d095cdb 100644 --- a/tests/covalent_dispatcher_tests/_core/runner_ng_test.py +++ b/tests/covalent_dispatcher_tests/_core/runner_ng_test.py @@ -318,12 +318,15 @@ async def test_get_task_result(mocker): "assets": { "output": { "remote_uri": asset_uri, + "size": 0, }, "stdout": { "remote_uri": asset_uri, + "size": 0, }, "stderr": { "remote_uri": asset_uri, + "size": 0, }, }, "status": RESULT_STATUS.COMPLETED, diff --git a/tests/covalent_dispatcher_tests/_dal/asset_test.py b/tests/covalent_dispatcher_tests/_dal/asset_test.py index 2f3975525..4a9f6fe78 100644 --- a/tests/covalent_dispatcher_tests/_dal/asset_test.py +++ b/tests/covalent_dispatcher_tests/_dal/asset_test.py @@ -36,7 +36,7 @@ def test_db(): ) -def get_asset_record(storage_path, object_key, digest_alg="", digest="", size=0): +def get_asset_record(storage_path, object_key, digest_alg="", digest="", size=1024): return models.Asset( storage_type=StorageType.LOCAL.value, storage_path=storage_path, @@ -61,7 +61,7 @@ def test_asset_load_data(): os.unlink(temppath) -def test_asset_store_data(): +def test_asset_store_data(test_db): with tempfile.NamedTemporaryFile("w", delete=True, suffix=".txt") as temp: temppath = temp.name key = os.path.basename(temppath) @@ -70,7 +70,8 @@ def test_asset_store_data(): rec = get_asset_record(storage_path, key) a = Asset(None, rec) - a.store_data("Hello\n") + with test_db.session() as session: + a.store_data("Hello\n", session) with open(temppath, "r") as f: assert f.read() == "Hello\n" @@ -78,7 +79,7 @@ def test_asset_store_data(): os.unlink(temppath) -def test_upload_asset(): +def test_upload_asset(test_db): with tempfile.NamedTemporaryFile("w", delete=True, suffix=".txt") as temp: src_path = temp.name src_key = os.path.basename(src_path) @@ -87,7 +88,8 @@ def test_upload_asset(): rec = get_asset_record(storage_path, src_key) a = Asset(None, rec) - a.store_data("Hello\n") + with test_db.session() as session: + a.store_data("Hello\n", session) with tempfile.NamedTemporaryFile("w", delete=True, suffix=".txt") as temp: dest_path = temp.name diff --git a/tests/covalent_dispatcher_tests/_dal/electron_test.py b/tests/covalent_dispatcher_tests/_dal/electron_test.py index f87451e8e..d05eb9529 100644 --- a/tests/covalent_dispatcher_tests/_dal/electron_test.py +++ b/tests/covalent_dispatcher_tests/_dal/electron_test.py @@ -265,7 +265,7 @@ def test_electron_update_assets(test_db, mocker): updates = {"output": {"size": 2048}} - e.update_assets(updates) + e.update_assets(updates, session) output = e.get_asset("output", session) output.refresh(session, fields={"size"}, for_update=False) diff --git a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py index b612e8ac8..440742cba 100644 --- a/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py +++ b/tests/covalent_dispatcher_tests/_dal/importers/result_import_test.py @@ -230,9 +230,9 @@ def test_import_result_with_custom_assets(mocker, test_db): prefix="covalent-" ) as srv_dir: manifest = get_mock_result(dispatch_id, sdk_dir) - manifest.lattice.custom_assets = {"custom_lattice_asset": AssetSchema()} + manifest.lattice.custom_assets = {"custom_lattice_asset": AssetSchema(size=0)} manifest.lattice.transport_graph.nodes[0].custom_assets = { - "custom_electron_asset": AssetSchema() + "custom_electron_asset": AssetSchema(size=0) } filtered_res = import_result(manifest, srv_dir, None) diff --git a/tests/covalent_dispatcher_tests/_db/update_test.py b/tests/covalent_dispatcher_tests/_db/update_test.py index 1d9230f67..b02875e0c 100644 --- a/tests/covalent_dispatcher_tests/_db/update_test.py +++ b/tests/covalent_dispatcher_tests/_db/update_test.py @@ -141,18 +141,15 @@ def test_result_persist_workflow_1(test_db, result_1, mocker): storage_path=lattice_storage_path, filename=lattice_row.function_filename ).get_deserialized() assert workflow_function(1, 2) == 4 - assert ( + with pytest.raises(FileNotFoundError): local_store.load_file( storage_path=lattice_storage_path, filename=lattice_row.error_filename ) - == "" - ) - assert ( + + with pytest.raises(FileNotFoundError): local_store.load_file( storage_path=lattice_storage_path, filename=lattice_row.results_filename - ).get_deserialized() - is None - ) + ) executor_data = json.loads(lattice_row.executor_data) diff --git a/tests/covalent_dispatcher_tests/_db/upsert_test.py b/tests/covalent_dispatcher_tests/_db/upsert_test.py index 2618870bc..63418c66d 100644 --- a/tests/covalent_dispatcher_tests/_db/upsert_test.py +++ b/tests/covalent_dispatcher_tests/_db/upsert_test.py @@ -83,7 +83,7 @@ def test_upsert_electron_data_handles_missing_keys(test_db, result_1, mocker): mocker.patch("covalent_dispatcher._db.write_result_to_db.workflow_db", test_db) mocker.patch("covalent_dispatcher._db.upsert.workflow_db", test_db) mock_store_file = mocker.patch( - "covalent_dispatcher._db.upsert.local_store.store_file", return_value=mock_digest + "covalent_dispatcher._db.upsert.local_store.store_file", return_value=(mock_digest, 2) ) mocker.patch( "covalent_dispatcher._db.upsert.Electron.meta_type.create", diff --git a/tests/covalent_dispatcher_tests/_object_store/local_test.py b/tests/covalent_dispatcher_tests/_object_store/local_test.py index 0823d3600..1b2f0ba7b 100644 --- a/tests/covalent_dispatcher_tests/_object_store/local_test.py +++ b/tests/covalent_dispatcher_tests/_object_store/local_test.py @@ -59,9 +59,11 @@ def test_store_and_load_file(): local_store.store_file(storage_path=temp_dir, filename="pickle.pkl", data=data) assert local_store.load_file(storage_path=temp_dir, filename="pickle.pkl") == data + # No file should be created in this case data = None local_store.store_file(storage_path=temp_dir, filename="pickle.txt", data=data) - assert local_store.load_file(storage_path=temp_dir, filename="pickle.txt") == "" + with pytest.raises(FileNotFoundError): + assert local_store.load_file(storage_path=temp_dir, filename="pickle.txt") data = b"test" local_store.store_file(storage_path=temp_dir, filename="pickle.mdb", data=data) diff --git a/tests/covalent_tests/dispatcher_plugins/local_test.py b/tests/covalent_tests/dispatcher_plugins/local_test.py index d34592ed7..d3c09c316 100644 --- a/tests/covalent_tests/dispatcher_plugins/local_test.py +++ b/tests/covalent_tests/dispatcher_plugins/local_test.py @@ -412,6 +412,8 @@ def workflow(a, b): num_assets = 0 # Populate the lattice asset schemas with dummy URLs for key, asset in manifest.lattice.assets: + if asset.size == 0: + continue num_assets += 1 asset.remote_uri = ( f"http://localhost:48008/api/v2/dispatches/{dispatch_id}/lattice/assets/dummy" @@ -420,11 +422,11 @@ def workflow(a, b): endpoint = f"/api/v2/dispatches/{dispatch_id}/lattice/assets/dummy" r = Response() r.status_code = 200 - mock_post = mocker.patch("covalent._api.apiclient.requests.Session.put", return_value=r) + mock_put = mocker.patch("covalent._api.apiclient.requests.Session.put", return_value=r) LocalDispatcher.upload_assets(manifest) - assert mock_post.call_count == num_assets + assert mock_put.call_count == num_assets def test_get_redispatch_request_body_norebuild(mocker): diff --git a/tests/covalent_tests/executor/executor_plugins/dask_test.py b/tests/covalent_tests/executor/executor_plugins/dask_test.py index c9b5c0886..548c35cf0 100644 --- a/tests/covalent_tests/executor/executor_plugins/dask_test.py +++ b/tests/covalent_tests/executor/executor_plugins/dask_test.py @@ -30,6 +30,7 @@ import covalent as ct from covalent._shared_files import TaskRuntimeError from covalent._shared_files.exceptions import TaskCancelledError +from covalent._shared_files.util_classes import RESULT_STATUS from covalent._workflow.transportable_object import TransportableObject from covalent.executor.executor_plugins.dask import ( _EXECUTOR_PLUGIN_DEFAULTS, @@ -375,6 +376,8 @@ async def run_dask_job(task_specs, resources, task_group_metadata): output = TransportableObject.deserialize(f.read()) assert output.get_deserialized() == 3 + assert task_update.assets["output"].size == os.path.getsize(output_uri) + def test_run_task_from_uris_alt(): """Test the wrapper submitted to dask""" @@ -615,3 +618,17 @@ def test_get_upload_uri(): assert dispatch_id in path_string assert str(task_group_id) in path_string assert object_key in path_string + + +@pytest.mark.asyncio +async def test_dask_receive_cancelled_tasks(): + dispatch_id = "mock_dispatch" + task_group_metadata = {"dispatch_id": dispatch_id, "node_ids": [0, 1], "task_group_id": 0} + dask_exec = DaskExecutor() + updates = await dask_exec.receive(task_group_metadata, None) + for task_update in updates: + assert task_update.status == RESULT_STATUS.CANCELLED + + for _, asset in task_update.assets.items(): + assert asset.remote_uri == "" + assert asset.size == 0 diff --git a/tests/functional_tests/local_executor_test.py b/tests/functional_tests/local_executor_test.py index 27db56c11..783c6d0ff 100644 --- a/tests/functional_tests/local_executor_test.py +++ b/tests/functional_tests/local_executor_test.py @@ -69,7 +69,7 @@ def workflow(a, b): dispatch_id = ct.dispatch(workflow)(a=1, b=2) workflow_result = rm.get_result(dispatch_id, wait=True) - assert workflow_result.error == "" + assert workflow_result.error is None assert workflow_result.status == Result.COMPLETED assert workflow_result.result == 3 assert workflow_result.get_node_result(node_id=0)["sublattice_result"].result == 3 diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index 7bab310bd..5d76993b3 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -118,7 +118,7 @@ def workflow(a, b): dispatch_id = ct.dispatch(workflow)(a=1, b=2) workflow_result = rm.get_result(dispatch_id, wait=True) - assert workflow_result.error == "" + assert workflow_result.error is None assert workflow_result.status == Result.COMPLETED assert workflow_result.result == 3 assert workflow_result.get_node_result(node_id=0)["sublattice_result"].result == 3 @@ -261,7 +261,7 @@ def workflow(file_path): dispatch_id = ct.dispatch(workflow)(file_path=tmp_path) res = ct.get_result(dispatch_id, wait=True) - assert res.error == "" + assert res.error is None assert res.result == (True, "Hello") From 60b09e141bb5ca510c2fac4d243283b492232f7b Mon Sep 17 00:00:00 2001 From: Casey Jao Date: Mon, 11 Dec 2023 10:18:39 -0500 Subject: [PATCH 4/6] Consolidate deps, call_before, call_after into `hooks` asset (#1880) --- CHANGELOG.md | 5 + covalent/_results_manager/results_manager.py | 8 +- covalent/_serialize/electron.py | 39 +- covalent/_serialize/lattice.py | 36 +- covalent/_shared_files/defaults.py | 7 +- covalent/_shared_files/schemas/electron.py | 14 +- covalent/_shared_files/schemas/lattice.py | 14 +- covalent/_workflow/electron.py | 43 +- covalent/_workflow/lattice.py | 38 +- covalent/_workflow/lepton.py | 30 +- covalent/_workflow/transport.py | 34 +- covalent/executor/executor_plugins/dask.py | 4 +- covalent/executor/executor_plugins/local.py | 5 +- covalent/executor/schemas.py | 9 +- covalent/executor/utils/wrappers.py | 70 +- covalent_dispatcher/_core/runner.py | 12 +- covalent_dispatcher/_core/runner_ng.py | 23 +- .../_dal/db_interfaces/lattice_utils.py | 4 +- .../_dal/importers/electron.py | 8 +- covalent_dispatcher/_dal/importers/lattice.py | 8 +- covalent_dispatcher/_db/models.py | 18 +- covalent_dispatcher/_db/upsert.py | 32 +- covalent_dispatcher/_db/write_result_to_db.py | 16 +- covalent_dispatcher/_service/models.py | 47 +- ...3727163f275c_consolidate_electron_hooks.py | 73 +++ covalent_ui/api/v1/data_layer/electron_dal.py | 4 +- .../api/v1/database/schema/electron.py | 12 +- .../api/v1/database/schema/lattices.py | 8 +- covalent_ui/api/v1/models/electrons_model.py | 4 +- .../v1/routes/end_points/electron_routes.py | 12 +- .../_core/runner_ng_test.py | 29 +- .../_db/update_test.py | 18 +- .../_db/write_result_to_db_test.py | 24 +- .../executor/executor_plugins/dask_test.py | 108 ++-- .../executor/executor_plugins/local_test.py | 99 +-- .../workflow/electron_metadata_test.py | 13 +- .../covalent_tests/workflow/electron_test.py | 22 +- tests/covalent_tests/workflow/lepton_test.py | 14 +- .../covalent_tests/workflow/transport_test.py | 18 +- .../end_points/electrons_test.py | 34 +- .../utils/assert_data/electrons.py | 24 +- .../utils/data/electrons.json | 608 +++++------------- .../utils/data/electrons_old.json | 608 +++++------------- .../utils/data/lattices.json | 20 +- .../utils/seed_script.py | 9 +- tests/functional_tests/workflow_stack_test.py | 7 +- 46 files changed, 700 insertions(+), 1592 deletions(-) create mode 100644 covalent_migrations/versions/3727163f275c_consolidate_electron_hooks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ffb8aca..ec165d88b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 asset metadata. Although the exact numerical value is not yet important, whether the size is reported to be zero or positive does have consequences. +- Pack deps, call_before, and call_after assets into one file. + +### Fixed + +- Reduced number of assets to upload when submitting a dispatch. ### Operations diff --git a/covalent/_results_manager/results_manager.py b/covalent/_results_manager/results_manager.py index 9372a43e0..e31670c90 100644 --- a/covalent/_results_manager/results_manager.py +++ b/covalent/_results_manager/results_manager.py @@ -51,9 +51,7 @@ SDK_NODE_META_KEYS = { "executor", "executor_data", - "deps", - "call_before", - "call_after", + "hooks", } SDK_LAT_META_KEYS = { @@ -61,9 +59,7 @@ "executor_data", "workflow_executor", "workflow_executor_data", - "deps", - "call_before", - "call_after", + "hooks", } DEFERRED_KEYS = { diff --git a/covalent/_serialize/electron.py b/covalent/_serialize/electron.py index 03a3b1ae2..fe5763675 100644 --- a/covalent/_serialize/electron.py +++ b/covalent/_serialize/electron.py @@ -39,9 +39,7 @@ "function_string": AssetType.TEXT, "value": AssetType.TRANSPORTABLE, "output": AssetType.TRANSPORTABLE, - "deps": AssetType.JSONABLE, - "call_before": AssetType.JSONABLE, - "call_after": AssetType.JSONABLE, + "hooks": AssetType.JSONABLE, "qelectron_db": AssetType.BYTES, "stdout": AssetType.TEXT, "stderr": AssetType.TEXT, @@ -160,27 +158,10 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron ASSET_FILENAME_MAP["error"], ) - deps = node_attrs["metadata"]["deps"] - deps_asset = save_asset( - deps, ASSET_TYPES["deps"], node_storage_path, ASSET_FILENAME_MAP["deps"] + hooks = node_attrs["metadata"]["hooks"] + hooks_asset = save_asset( + hooks, ASSET_TYPES["hooks"], node_storage_path, ASSET_FILENAME_MAP["hooks"] ) - - call_before = node_attrs["metadata"]["call_before"] - call_before_asset = save_asset( - call_before, - ASSET_TYPES["call_before"], - node_storage_path, - ASSET_FILENAME_MAP["call_before"], - ) - - call_after = node_attrs["metadata"]["call_after"] - call_after_asset = save_asset( - call_after, - ASSET_TYPES["call_after"], - node_storage_path, - ASSET_FILENAME_MAP["call_after"], - ) - return ElectronAssets( function=function_asset, function_string=function_string_asset, @@ -190,9 +171,7 @@ def _serialize_node_assets(node_attrs: dict, node_storage_path: str) -> Electron stderr=stderr_asset, qelectron_db=qelectron_db_asset, error=error_asset, - deps=deps_asset, - call_before=call_before_asset, - call_after=call_after_asset, + hooks=hooks_asset, ) @@ -206,9 +185,7 @@ def _deserialize_node_assets(ea: ElectronAssets) -> dict: qelectron_db = load_asset(ea.qelectron_db, ASSET_TYPES["qelectron_db"]) error = load_asset(ea.error, ASSET_TYPES["error"]) - deps = load_asset(ea.deps, ASSET_TYPES["deps"]) - call_before = load_asset(ea.call_before, ASSET_TYPES["call_before"]) - call_after = load_asset(ea.call_after, ASSET_TYPES["call_after"]) + hooks = load_asset(ea.hooks, ASSET_TYPES["hooks"]) return { "function": function, @@ -220,9 +197,7 @@ def _deserialize_node_assets(ea: ElectronAssets) -> dict: "qelectron_db": qelectron_db, "error": error, "metadata": { - "deps": deps, - "call_before": call_before, - "call_after": call_after, + "hooks": hooks, }, } diff --git a/covalent/_serialize/lattice.py b/covalent/_serialize/lattice.py index b188b8edf..3ab39f2bc 100644 --- a/covalent/_serialize/lattice.py +++ b/covalent/_serialize/lattice.py @@ -44,9 +44,7 @@ "named_kwargs": AssetType.TRANSPORTABLE, "cova_imports": AssetType.JSONABLE, "lattice_imports": AssetType.TEXT, - "deps": AssetType.JSONABLE, - "call_before": AssetType.JSONABLE, - "call_after": AssetType.JSONABLE, + "hooks": AssetType.JSONABLE, } @@ -141,23 +139,11 @@ def _serialize_lattice_assets(lat, storage_path: str) -> LatticeAssets: ) # NOTE: these are actually JSONable - deps_asset = save_asset( - lat.metadata["deps"], - ASSET_TYPES["deps"], + hooks_asset = save_asset( + lat.metadata["hooks"], + ASSET_TYPES["hooks"], storage_path, - ASSET_FILENAME_MAP["deps"], - ) - call_before_asset = save_asset( - lat.metadata["call_before"], - ASSET_TYPES["call_before"], - storage_path, - ASSET_FILENAME_MAP["call_before"], - ) - call_after_asset = save_asset( - lat.metadata["call_after"], - ASSET_TYPES["call_after"], - storage_path, - ASSET_FILENAME_MAP["call_after"], + ASSET_FILENAME_MAP["hooks"], ) return LatticeAssets( @@ -169,9 +155,7 @@ def _serialize_lattice_assets(lat, storage_path: str) -> LatticeAssets: named_kwargs=named_kwargs_asset, cova_imports=cova_imports_asset, lattice_imports=lattice_imports_asset, - deps=deps_asset, - call_before=call_before_asset, - call_after=call_after_asset, + hooks=hooks_asset, ) @@ -186,9 +170,7 @@ def _deserialize_lattice_assets(assets: LatticeAssets) -> dict: named_kwargs = load_asset(assets.named_kwargs, ASSET_TYPES["named_kwargs"]) cova_imports = load_asset(assets.cova_imports, ASSET_TYPES["cova_imports"]) lattice_imports = load_asset(assets.lattice_imports, ASSET_TYPES["lattice_imports"]) - deps = load_asset(assets.deps, ASSET_TYPES["deps"]) - call_before = load_asset(assets.call_before, ASSET_TYPES["call_before"]) - call_after = load_asset(assets.call_after, ASSET_TYPES["call_after"]) + hooks = load_asset(assets.hooks, ASSET_TYPES["hooks"]) return { "workflow_function": workflow_function, "workflow_function_string": workflow_function_string, @@ -199,9 +181,7 @@ def _deserialize_lattice_assets(assets: LatticeAssets) -> dict: "cova_imports": cova_imports, "lattice_imports": lattice_imports, "metadata": { - "deps": deps, - "call_before": call_before, - "call_after": call_after, + "hooks": hooks, }, } diff --git a/covalent/_shared_files/defaults.py b/covalent/_shared_files/defaults.py index 3fdcaf010..a1b9983a4 100644 --- a/covalent/_shared_files/defaults.py +++ b/covalent/_shared_files/defaults.py @@ -17,9 +17,8 @@ """Create custom sentinels and defaults for Covalent""" import os -from builtins import list from dataclasses import dataclass, field -from typing import Dict, List +from typing import Dict import dask.system @@ -199,8 +198,6 @@ class DefaultConfig: class DefaultMetadataValues: executor: str = field(default_factory=get_default_executor) executor_data: Dict = field(default_factory=dict) - deps: Dict = field(default_factory=dict) - call_before: List = field(default_factory=list) - call_after: List = field(default_factory=list) + hooks: Dict = field(default_factory=dict) workflow_executor: str = field(default_factory=get_default_executor) workflow_executor_data: Dict = field(default_factory=dict) diff --git a/covalent/_shared_files/schemas/electron.py b/covalent/_shared_files/schemas/electron.py index d75500d82..4a6541383 100644 --- a/covalent/_shared_files/schemas/electron.py +++ b/covalent/_shared_files/schemas/electron.py @@ -46,9 +46,7 @@ "stderr", "qelectron_db", # user dependent assets - "deps", - "call_before", - "call_after", + "hooks", } ELECTRON_FUNCTION_FILENAME = "function.tobj" @@ -61,7 +59,7 @@ ) ELECTRON_ERROR_FILENAME = "error.log" ELECTRON_RESULTS_FILENAME = "results.tobj" -ELECTRON_DEPS_FILENAME = "deps.json" +ELECTRON_HOOKS_FILENAME = "hooks.json" ELECTRON_CALL_BEFORE_FILENAME = "call_before.json" ELECTRON_CALL_AFTER_FILENAME = "call_after.json" ELECTRON_STORAGE_TYPE = "file" @@ -76,9 +74,7 @@ "stderr": ELECTRON_STDERR_FILENAME, "qelectron_db": ELECTRON_QELECTRON_DB_FILENAME, "error": ELECTRON_ERROR_FILENAME, - "deps": ELECTRON_DEPS_FILENAME, - "call_before": ELECTRON_CALL_BEFORE_FILENAME, - "call_after": ELECTRON_CALL_AFTER_FILENAME, + "hooks": ELECTRON_HOOKS_FILENAME, } @@ -93,9 +89,7 @@ class ElectronAssets(BaseModel): qelectron_db: AssetSchema # user dependent assets - deps: AssetSchema - call_before: AssetSchema - call_after: AssetSchema + hooks: AssetSchema class ElectronMetadata(BaseModel): diff --git a/covalent/_shared_files/schemas/lattice.py b/covalent/_shared_files/schemas/lattice.py index 80a8c62aa..0fdf35f16 100644 --- a/covalent/_shared_files/schemas/lattice.py +++ b/covalent/_shared_files/schemas/lattice.py @@ -44,9 +44,7 @@ "cova_imports", "lattice_imports", # user dependent assets - "deps", - "call_before", - "call_after", + "hooks", } LATTICE_FUNCTION_FILENAME = "function.tobj" @@ -59,7 +57,7 @@ LATTICE_NAMED_ARGS_FILENAME = "named_args.tobj" LATTICE_NAMED_KWARGS_FILENAME = "named_kwargs.tobj" LATTICE_RESULTS_FILENAME = "results.tobj" -LATTICE_DEPS_FILENAME = "deps.json" +LATTICE_HOOKS_FILENAME = "hooks.json" LATTICE_CALL_BEFORE_FILENAME = "call_before.json" LATTICE_CALL_AFTER_FILENAME = "call_after.json" LATTICE_COVA_IMPORTS_FILENAME = "cova_imports.json" @@ -76,9 +74,7 @@ "named_kwargs": LATTICE_NAMED_KWARGS_FILENAME, "cova_imports": LATTICE_COVA_IMPORTS_FILENAME, "lattice_imports": LATTICE_LATTICE_IMPORTS_FILENAME, - "deps": LATTICE_DEPS_FILENAME, - "call_before": LATTICE_CALL_BEFORE_FILENAME, - "call_after": LATTICE_CALL_AFTER_FILENAME, + "hooks": LATTICE_HOOKS_FILENAME, } @@ -93,9 +89,7 @@ class LatticeAssets(BaseModel): lattice_imports: AssetSchema # lattice.metadata - deps: AssetSchema - call_before: AssetSchema - call_after: AssetSchema + hooks: AssetSchema class LatticeMetadata(BaseModel): diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index ebe313675..b772a8954 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -16,12 +16,12 @@ """Class corresponding to computation nodes.""" - import inspect import json import operator import tempfile from builtins import list +from copy import deepcopy from dataclasses import asdict from functools import wraps from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Union @@ -306,13 +306,16 @@ def get_item(e, key): get_item.__name__ = node_name - iterable_metadata = self.metadata.copy() + # Perform a deep copy so as to not modify the parent + # electron's hooks + iterable_metadata = deepcopy(self.metadata) filtered_call_before = [] - for elem in iterable_metadata["call_before"]: - if elem["attributes"]["retval_keyword"] != "files": - filtered_call_before.append(elem) - iterable_metadata["call_before"] = filtered_call_before + if "call_before" in iterable_metadata["hooks"]: + for elem in iterable_metadata["hooks"]["call_before"]: + if elem["attributes"]["retval_keyword"] != "files": + filtered_call_before.append(elem) + iterable_metadata["hooks"]["call_before"] = filtered_call_before # Pack with main electron unless it is a sublattice. name = active_lattice.transport_graph.get_node_value(self.node_id, "name") @@ -452,6 +455,7 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: # Add a node to the transport graph of the active lattice. Electrons bound to nodes will never be packed with the # 'master' Electron. # Add non-sublattice node to the transport graph of the active lattice. + self.node_id = active_lattice.transport_graph.add_node( name=self.function.__name__, function=self.function, @@ -474,15 +478,20 @@ def __call__(self, *args, **kwargs) -> Union[Any, "Electron"]: # For keyword arguments # Filter out kwargs to be injected by call_before call_deps during execution. - call_before = self.metadata["call_before"] - retval_keywords = {item["attributes"]["retval_keyword"]: None for item in call_before} + + retval_keywords = {} + if "call_before" in self.metadata["hooks"]: + call_before = self.metadata["hooks"]["call_before"] + retval_keywords = { + item["attributes"]["retval_keyword"]: None for item in call_before + } + for key, value in named_kwargs.items(): if key in retval_keywords: app_log.debug( f"kwarg {key} for function {self.function.__name__} to be injected at runtime" ) continue - self.connect_node_with_others( self.node_id, key, value, "kwarg", None, active_lattice.transport_graph ) @@ -771,13 +780,21 @@ def electron( if call_after: call_after_final.extend(call_after) + if deps is None and call_before_final is None and call_after_final is None: + hooks = None + else: + hooks = {} + if deps is not None: + hooks["deps"] = deps + if call_before_final is not None: + hooks["call_before"] = call_before_final + if call_after_final is not None: + hooks["call_after"] = call_after_final + constraints = { "executor": executor, - "deps": deps, - "call_before": call_before_final, - "call_after": call_after_final, + "hooks": hooks, } - constraints = encode_metadata(constraints) def decorator_electron(func=None): diff --git a/covalent/_workflow/lattice.py b/covalent/_workflow/lattice.py index 2c4159d99..8f81fb706 100644 --- a/covalent/_workflow/lattice.py +++ b/covalent/_workflow/lattice.py @@ -18,11 +18,9 @@ import importlib.metadata import json -import os import warnings import webbrowser from builtins import list -from contextlib import redirect_stdout from copy import deepcopy from dataclasses import asdict from functools import wraps @@ -211,7 +209,7 @@ def build_graph(self, *args, **kwargs) -> None: self.lattice_imports, self.cova_imports = get_imports(workflow_function) # Set any lattice metadata not explicitly set by the user - constraint_names = {"executor", "workflow_executor", "deps", "call_before", "call_after"} + constraint_names = {"executor", "workflow_executor", "hooks"} new_metadata = { name: DEFAULT_METADATA_VALUES[name] for name in constraint_names @@ -225,15 +223,14 @@ def build_graph(self, *args, **kwargs) -> None: # Check whether task packing is enabled self._task_packing = get_config("sdk.task_packing") == "true" - with redirect_stdout(open(os.devnull, "w")): - with active_lattice_manager.claim(self): - try: - retval = workflow_function(*new_args, **new_kwargs) - except Exception: - warnings.warn( - "Please make sure you are not manipulating an object inside the lattice." - ) - raise + with active_lattice_manager.claim(self): + try: + retval = workflow_function(*new_args, **new_kwargs) + except Exception: + warnings.warn( + "Please make sure you are not manipulating an object inside the lattice." + ) + raise pp = Postprocessor(lattice=self) @@ -364,7 +361,7 @@ def lattice( ) executor = backend - deps = {} + deps = {} if deps_bash or deps_pip else None if isinstance(deps_bash, DepsBash): deps["bash"] = deps_bash @@ -387,12 +384,21 @@ def lattice( if isinstance(triggers, BaseTrigger): triggers = [triggers] + if deps is None and call_before is None and call_after is None: + hooks = None + else: + hooks = {} + if deps is not None: + hooks["deps"] = deps + if call_before is not None: + hooks["call_before"] = call_before + if call_after is not None: + hooks["call_after"] = call_after + constraints = { "executor": executor, "workflow_executor": workflow_executor, - "deps": deps, - "call_before": call_before, - "call_after": call_after, + "hooks": hooks, "triggers": triggers, } diff --git a/covalent/_workflow/lepton.py b/covalent/_workflow/lepton.py index f9084f2c8..a996ebeed 100644 --- a/covalent/_workflow/lepton.py +++ b/covalent/_workflow/lepton.py @@ -41,6 +41,11 @@ # TODO: Review exceptions/errors +DEFAULT_HOOKS = DEFAULT_METADATA_VALUES["hooks"] +DEFAULT_DEPS = DEFAULT_HOOKS.get("deps", {}) +DEFAULT_CALL_BEFORE = DEFAULT_HOOKS.get("call_before", []) +DEFAULT_CALL_AFTER = DEFAULT_HOOKS.get("call_after", []) + class Lepton(Electron): """ @@ -86,10 +91,10 @@ def __init__( List[Union[str, "BaseExecutor"]], Union[str, "BaseExecutor"] ] = DEFAULT_METADATA_VALUES["executor"], files: List[FileTransfer] = [], - deps_bash: Union[DepsBash, List, str] = DEFAULT_METADATA_VALUES["deps"].get("bash", []), - deps_pip: Union[DepsPip, list] = DEFAULT_METADATA_VALUES["deps"].get("pip", None), - call_before: Union[List[DepsCall], DepsCall] = DEFAULT_METADATA_VALUES["call_before"], - call_after: Union[List[DepsCall], DepsCall] = DEFAULT_METADATA_VALUES["call_after"], + deps_bash: Union[DepsBash, List, str] = DEFAULT_DEPS.get("bash"), + deps_pip: Union[DepsPip, list] = DEFAULT_DEPS.get("pip"), + call_before: Union[List[DepsCall], DepsCall] = DEFAULT_CALL_BEFORE, + call_after: Union[List[DepsCall], DepsCall] = DEFAULT_CALL_AFTER, ) -> None: self.language = language self.library_name = library_name @@ -174,13 +179,16 @@ def __init__( "DepsCall retval_keyword(s) are not currently supported for Leptons, please remove the retval_keyword arg from DepsCall for the workflow to be constructed successfully." ) - # Should be synced with electron - constraints = { - "executor": executor, + hooks = { "deps": deps, "call_before": call_before, "call_after": call_after, } + # Should be synced with electron + constraints = { + "executor": executor, + "hooks": hooks, + } constraints = encode_metadata(constraints) @@ -424,10 +432,10 @@ def bash( Union[List[Union[str, "BaseExecutor"]], Union[str, "BaseExecutor"]] ] = DEFAULT_METADATA_VALUES["executor"], files: List[FileTransfer] = [], - deps_bash: Union[DepsBash, List, str] = DEFAULT_METADATA_VALUES["deps"].get("bash", []), - deps_pip: Union[DepsPip, list] = DEFAULT_METADATA_VALUES["deps"].get("pip", None), - call_before: Union[List[DepsCall], DepsCall] = DEFAULT_METADATA_VALUES["call_before"], - call_after: Union[List[DepsCall], DepsCall] = DEFAULT_METADATA_VALUES["call_after"], + deps_bash: Union[DepsBash, List, str] = DEFAULT_DEPS.get("bash", []), + deps_pip: Union[DepsPip, list] = DEFAULT_DEPS.get("pip", []), + call_before: Union[List[DepsCall], DepsCall] = DEFAULT_CALL_BEFORE, + call_after: Union[List[DepsCall], DepsCall] = DEFAULT_CALL_AFTER, ) -> Callable: """Bash decorator which wraps a Python function as a Bash Lepton.""" diff --git a/covalent/_workflow/transport.py b/covalent/_workflow/transport.py index c1c7dd31d..2ab1db739 100644 --- a/covalent/_workflow/transport.py +++ b/covalent/_workflow/transport.py @@ -56,21 +56,25 @@ def encode_metadata(metadata: dict) -> dict: encoded_metadata["workflow_executor_data"] = encoded_wf_executor # Bash Deps, Pip Deps, Env Deps, etc - if "deps" in metadata and metadata["deps"] is not None: - for dep_type, dep_object in metadata["deps"].items(): - if dep_object and not isinstance(dep_object, dict): - encoded_metadata["deps"][dep_type] = dep_object.to_dict() - - # call_before/after - if "call_before" in metadata and metadata["call_before"] is not None: - for i, dep in enumerate(metadata["call_before"]): - if not isinstance(dep, dict): - encoded_metadata["call_before"][i] = dep.to_dict() - - if "call_after" in metadata and metadata["call_after"] is not None: - for i, dep in enumerate(metadata["call_after"]): - if not isinstance(dep, dict): - encoded_metadata["call_after"][i] = dep.to_dict() + + if "hooks" in metadata and metadata["hooks"] is not None: + hooks = metadata["hooks"] + + if "deps" in hooks and hooks["deps"] is not None: + for dep_type, dep_object in hooks["deps"].items(): + if dep_object and not isinstance(dep_object, dict): + encoded_metadata["hooks"]["deps"][dep_type] = dep_object.to_dict() + + # call_before/after + if "call_before" in hooks and hooks["call_before"] is not None: + for i, dep in enumerate(hooks["call_before"]): + if not isinstance(dep, dict): + encoded_metadata["hooks"]["call_before"][i] = dep.to_dict() + + if "call_after" in hooks and hooks["call_after"] is not None: + for i, dep in enumerate(hooks["call_after"]): + if not isinstance(dep, dict): + encoded_metadata["hooks"]["call_after"][i] = dep.to_dict() # triggers if "triggers" in metadata: diff --git a/covalent/executor/executor_plugins/dask.py b/covalent/executor/executor_plugins/dask.py index fa7c8c315..64d52cb04 100644 --- a/covalent/executor/executor_plugins/dask.py +++ b/covalent/executor/executor_plugins/dask.py @@ -40,7 +40,7 @@ from covalent.executor.base import AsyncBaseExecutor from covalent.executor.schemas import ResourceMap, TaskSpec, TaskUpdate from covalent.executor.utils.wrappers import io_wrapper as dask_wrapper -from covalent.executor.utils.wrappers import run_task_from_uris_alt +from covalent.executor.utils.wrappers import run_task_group_alt # The plugin class name must be given by the executor_plugin_name attribute: EXECUTOR_PLUGIN_NAME = "DaskExecutor" @@ -261,7 +261,7 @@ async def send( await self.set_job_handle(key) future = dask_client.submit( - run_task_from_uris_alt, + run_task_group_alt, list(map(lambda t: t.model_dump(), task_specs)), resources.model_dump(), output_uris, diff --git a/covalent/executor/executor_plugins/local.py b/covalent/executor/executor_plugins/local.py index 38047d040..c85352708 100644 --- a/covalent/executor/executor_plugins/local.py +++ b/covalent/executor/executor_plugins/local.py @@ -40,7 +40,7 @@ # Store the wrapper function in an external module to avoid module # import errors during pickling -from covalent.executor.utils.wrappers import io_wrapper, run_task_from_uris +from covalent.executor.utils.wrappers import io_wrapper, run_task_group # The plugin class name must be given by the executor_plugin_name attribute: EXECUTOR_PLUGIN_NAME = "LocalExecutor" @@ -178,9 +178,8 @@ def _send( app_log.debug(f"Running task group {dispatch_id}:{task_ids}") future = proc_pool.submit( - run_task_from_uris, + run_task_group, list(map(lambda t: t.model_dump(), task_specs)), - resources.model_dump(), output_uris, self.cache_dir, task_group_metadata, diff --git a/covalent/executor/schemas.py b/covalent/executor/schemas.py index 2e9b16204..40b4c732e 100644 --- a/covalent/executor/schemas.py +++ b/covalent/executor/schemas.py @@ -56,9 +56,7 @@ class TaskSpec(BaseModel): function_id: The `node_id` of the function. args_ids: The `node_id`s of the function's args kwargs_ids: The `node_id`s of the function's kwargs {key: node_id} - deps_id: An opaque string representing the task's deps. - call_before_id: An opaque string representing the task's call_before. - call_after_id: An opaque string representing the task's call_before. + hooks_id: An opaque string representing the task's hooks. The attribute values can be used in conjunction with a `ResourceMap` to locate the actual resources in the compute @@ -68,9 +66,6 @@ class TaskSpec(BaseModel): function_id: int args_ids: List[int] kwargs_ids: Dict[str, int] - deps_id: str - call_before_id: str - call_after_id: str class ResourceMap(BaseModel): @@ -101,7 +96,7 @@ class ResourceMap(BaseModel): inputs: Dict[int, str] # Includes deps, call_before, call_after - deps: Dict[str, str] + hooks: Dict[int, str] class TaskGroup(BaseModel): diff --git a/covalent/executor/utils/wrappers.py b/covalent/executor/utils/wrappers.py index 90dec9e64..4cbd7fccb 100644 --- a/covalent/executor/utils/wrappers.py +++ b/covalent/executor/utils/wrappers.py @@ -155,17 +155,15 @@ def _gather_deps(deps, call_before_objs_json, call_after_objs_json) -> Tuple[Lis # for `AsyncBaseExecutor.send()`. -# URIs are just file paths -def run_task_from_uris( +def run_task_group( task_specs: List[Dict], - resources: dict, output_uris: List[Tuple[str, str, str]], results_dir: str, task_group_metadata: dict, server_url: str, ): """ - Run the task from URIs. + Run a task group. This is appropriate for executors which can access the Covalent server url directly. Exampl: LocalExecutor. @@ -220,21 +218,14 @@ def run_task_from_uris( resp.raise_for_status() ser_kwargs[k] = deserialize_node_asset(resp.content, "output") - # Download deps - deps_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/{task_id}/assets/deps" - resp = requests.get(deps_url, stream=True) - resp.raise_for_status() - deps_json = deserialize_node_asset(resp.content, "deps") - - cb_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/{task_id}/assets/call_before" - resp = requests.get(cb_url, stream=True) - resp.raise_for_status() - call_before_json = deserialize_node_asset(resp.content, "call_before") - - ca_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/{task_id}/assets/call_after" - resp = requests.get(ca_url, stream=True) + # Download deps, call_before, and call_after + hooks_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/{task_id}/assets/hooks" + resp = requests.get(hooks_url, stream=True) resp.raise_for_status() - call_after_json = deserialize_node_asset(resp.content, "call_after") + hooks_json = deserialize_node_asset(resp.content, "hooks") + deps_json = hooks_json.get("deps", {}) + call_before_json = hooks_json.get("call_before", []) + call_after_json = hooks_json.get("call_after", []) # Assemble and run the task call_before, call_after = _gather_deps( @@ -345,8 +336,7 @@ def run_task_from_uris( requests.put(url) -# URIs are just file paths -def run_task_from_uris_alt( +def run_task_group_alt( task_specs: List[Dict], resources: dict, output_uris: List[Tuple[str, str, str]], @@ -355,7 +345,7 @@ def run_task_from_uris_alt( server_url: str, ): """ - Alternate form of run_task_from_uris for sync executors. + Alternate form of run_task_group. This is appropriate for backends that cannot reach the Covalent server. Covalent will push input assets to the executor's @@ -417,27 +407,15 @@ def run_task_from_uris_alt( with open(uri, "rb") as f: ser_kwargs[key] = deserialize_node_asset(f.read(), "output") - # Load deps - deps_id = task["deps_id"] - deps_uri = resources["deps"][deps_id] - if deps_uri.startswith(prefix): - deps_uri = deps_uri[prefix_len:] - with open(deps_uri, "rb") as f: - deps_json = deserialize_node_asset(f.read(), "deps") - - call_before_id = task["call_before_id"] - call_before_uri = resources["deps"][call_before_id] - if call_before_uri.startswith(prefix): - call_before_uri = call_before_uri[prefix_len:] - with open(call_before_uri, "rb") as f: - call_before_json = deserialize_node_asset(f.read(), "call_before") - - call_after_id = task["call_after_id"] - call_after_uri = resources["deps"][call_after_id] - if call_after_uri.startswith(prefix): - call_after_uri = call_after_uri[prefix_len:] - with open(call_after_uri, "rb") as f: - call_after_json = deserialize_node_asset(f.read(), "call_after") + # Load deps, call_before, and call_after + hooks_uri = resources["hooks"][task_id] + if hooks_uri.startswith(prefix): + hooks_uri = hooks_uri[prefix_len:] + with open(hooks_uri, "rb") as f: + hooks_json = deserialize_node_asset(f.read(), "hooks") + deps_json = hooks_json.get("deps", {}) + call_before_json = hooks_json.get("call_before", []) + call_after_json = hooks_json.get("call_after", []) # Assemble and invoke the task call_before, call_after = _gather_deps( @@ -500,16 +478,14 @@ def run_task_from_uris_alt( exception_occurred = True tb = "".join(traceback.TracebackException.from_exception(ex).format()) print(tb, file=sys.stderr) - result_uri = "" stdout.flush() stderr.flush() stdout_size = os.path.getsize(stdout_uri) stderr_size = os.path.getsize(stderr_uri) - qelectron_db_size = len(qelectron_db_bytes) result_summary = { "node_id": task_id, "output": { - "uri": result_uri, + "uri": "", "size": 0, }, "stdout": { @@ -521,8 +497,8 @@ def run_task_from_uris_alt( "size": stderr_size, }, "qelectron_db": { - "uri": qelectron_db_uri, - "size": qelectron_db_size, + "uri": "", + "size": 0, }, "exception_occurred": exception_occurred, } diff --git a/covalent_dispatcher/_core/runner.py b/covalent_dispatcher/_core/runner.py index b38cc1058..7477b61c0 100644 --- a/covalent_dispatcher/_core/runner.py +++ b/covalent_dispatcher/_core/runner.py @@ -218,16 +218,14 @@ async def _run_task( async def _gather_deps(dispatch_id: str, node_id: int) -> Tuple[List, List]: """Assemble deps for a node into the final call_before and call_after""" - deps_attrs = await datasvc.electron.get( - dispatch_id, node_id, ["deps", "call_before", "call_after"] - ) + resp = await datasvc.electron.get(dispatch_id, node_id, ["hooks"]) - deps = deps_attrs["deps"] + hooks = resp["hooks"] # Assemble call_before and call_after from all the deps - - call_before_objs_json = deps_attrs["call_before"] - call_after_objs_json = deps_attrs["call_after"] + deps = hooks.get("deps", {}) + call_before_objs_json = hooks.get("call_before", []) + call_after_objs_json = hooks.get("call_after", []) call_before = [] call_after = [] diff --git a/covalent_dispatcher/_core/runner_ng.py b/covalent_dispatcher/_core/runner_ng.py index b3988de88..5c98a7721 100644 --- a/covalent_dispatcher/_core/runner_ng.py +++ b/covalent_dispatcher/_core/runner_ng.py @@ -85,35 +85,20 @@ async def _submit_abstract_task_group( if not type(executor).SUPPORTS_MANAGED_EXECUTION: raise NotImplementedError("Executor does not support managed execution") - resources = {"functions": {}, "inputs": {}, "deps": {}} + resources = {"functions": {}, "inputs": {}, "hooks": {}} # Get upload URIs for task_spec in task_seq: task_id = task_spec["function_id"] function_uri = executor.get_upload_uri(task_group_metadata, f"function-{task_id}") - deps_uri = executor.get_upload_uri(task_group_metadata, f"deps-{task_id}") - call_before_uri = executor.get_upload_uri( - task_group_metadata, f"call_before-{task_id}" - ) - call_after_uri = executor.get_upload_uri(task_group_metadata, f"call_after-{task_id}") + hooks_uri = executor.get_upload_uri(task_group_metadata, f"hooks-{task_id}") await am.upload_asset_for_nodes(dispatch_id, "function", {task_id: function_uri}) - await am.upload_asset_for_nodes(dispatch_id, "deps", {task_id: deps_uri}) - await am.upload_asset_for_nodes(dispatch_id, "call_before", {task_id: call_before_uri}) - await am.upload_asset_for_nodes(dispatch_id, "call_after", {task_id: call_after_uri}) - - deps_id = f"deps-{task_id}" - call_before_id = f"call_before-{task_id}" - call_after_id = f"call_after-{task_id}" - task_spec["deps_id"] = deps_id - task_spec["call_before_id"] = call_before_id - task_spec["call_after_id"] = call_after_id + await am.upload_asset_for_nodes(dispatch_id, "hooks", {task_id: hooks_uri}) resources["functions"][task_id] = function_uri - resources["deps"][deps_id] = deps_uri - resources["deps"][call_before_id] = call_before_uri - resources["deps"][call_after_id] = call_after_uri + resources["hooks"][task_id] = hooks_uri task_specs.append(TaskSpec(**task_spec)) diff --git a/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py b/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py index 7d941d7b6..676d0b68c 100644 --- a/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py +++ b/covalent_dispatcher/_dal/db_interfaces/lattice_utils.py @@ -74,9 +74,7 @@ "lattice_imports": "lattice_imports_filename", "executor_data": "executor_data_filename", "workflow_executor_data": "workflow_executor_data_filename", - "deps": "deps_filename", - "call_before": "call_before_filename", - "call_after": "call_after_filename", + "hooks": "hooks_filename", } diff --git a/covalent_dispatcher/_dal/importers/electron.py b/covalent_dispatcher/_dal/importers/electron.py index 60b1a3e0d..d4b5047c5 100644 --- a/covalent_dispatcher/_dal/importers/electron.py +++ b/covalent_dispatcher/_dal/importers/electron.py @@ -26,12 +26,10 @@ from covalent._shared_files import logger from covalent._shared_files.schemas.electron import ( ASSET_FILENAME_MAP, - ELECTRON_CALL_AFTER_FILENAME, - ELECTRON_CALL_BEFORE_FILENAME, - ELECTRON_DEPS_FILENAME, ELECTRON_ERROR_FILENAME, ELECTRON_FUNCTION_FILENAME, ELECTRON_FUNCTION_STRING_FILENAME, + ELECTRON_HOOKS_FILENAME, ELECTRON_RESULTS_FILENAME, ELECTRON_STDERR_FILENAME, ELECTRON_STDOUT_FILENAME, @@ -111,9 +109,7 @@ def _get_electron_meta( "stdout_filename": ELECTRON_STDOUT_FILENAME, "stderr_filename": ELECTRON_STDERR_FILENAME, "error_filename": ELECTRON_ERROR_FILENAME, - "deps_filename": ELECTRON_DEPS_FILENAME, - "call_before_filename": ELECTRON_CALL_BEFORE_FILENAME, - "call_after_filename": ELECTRON_CALL_AFTER_FILENAME, + "hooks_filename": ELECTRON_HOOKS_FILENAME, } kwargs.update(legacy_kwargs) return kwargs diff --git a/covalent_dispatcher/_dal/importers/lattice.py b/covalent_dispatcher/_dal/importers/lattice.py index 7c5870b30..a14938f98 100644 --- a/covalent_dispatcher/_dal/importers/lattice.py +++ b/covalent_dispatcher/_dal/importers/lattice.py @@ -24,14 +24,12 @@ from covalent._shared_files.config import get_config from covalent._shared_files.schemas.lattice import ( - LATTICE_CALL_AFTER_FILENAME, - LATTICE_CALL_BEFORE_FILENAME, LATTICE_COVA_IMPORTS_FILENAME, - LATTICE_DEPS_FILENAME, LATTICE_DOCSTRING_FILENAME, LATTICE_ERROR_FILENAME, LATTICE_FUNCTION_FILENAME, LATTICE_FUNCTION_STRING_FILENAME, + LATTICE_HOOKS_FILENAME, LATTICE_INPUTS_FILENAME, LATTICE_LATTICE_IMPORTS_FILENAME, LATTICE_NAMED_ARGS_FILENAME, @@ -76,9 +74,7 @@ def _get_lattice_meta(lat: LatticeSchema, storage_path) -> dict: "named_args_filename": LATTICE_NAMED_ARGS_FILENAME, "named_kwargs_filename": LATTICE_NAMED_KWARGS_FILENAME, "results_filename": LATTICE_RESULTS_FILENAME, - "deps_filename": LATTICE_DEPS_FILENAME, - "call_before_filename": LATTICE_CALL_BEFORE_FILENAME, - "call_after_filename": LATTICE_CALL_AFTER_FILENAME, + "hooks_filename": LATTICE_HOOKS_FILENAME, "cova_imports_filename": LATTICE_COVA_IMPORTS_FILENAME, "lattice_imports_filename": LATTICE_LATTICE_IMPORTS_FILENAME, } diff --git a/covalent_dispatcher/_db/models.py b/covalent_dispatcher/_db/models.py index b5d1fe408..7e0521c35 100644 --- a/covalent_dispatcher/_db/models.py +++ b/covalent_dispatcher/_db/models.py @@ -101,14 +101,8 @@ class Lattice(Base): # name of the file containing the serialized output results_filename = Column(Text) - # Name of the file containing the default electron dependencies - deps_filename = Column(Text) - - # Name of the file containing the default list of callables before electrons are executed - call_before_filename = Column(Text) - - # Name of the file containing the default list of callables after electrons are executed - call_after_filename = Column(Text) + # Name of the file containing the default electron hooks + hooks_filename = Column(Text) # Name of the file containing the set of cova imports cova_imports_filename = Column(Text) @@ -189,13 +183,7 @@ class Electron(Base): stdout_filename = Column(Text) # Name of the file containing the electron execution dependencies - deps_filename = Column(Text) - - # Name of the file containing the functions that are called before electron execution - call_before_filename = Column(Text) - - # Name of the file containing the functions that are called before electron execution - call_after_filename = Column(Text) + hooks_filename = Column(Text) # Whether qelectron data exists or not qelectron_data_exists = Column(Boolean, nullable=False, default=False) diff --git a/covalent_dispatcher/_db/upsert.py b/covalent_dispatcher/_db/upsert.py index cc2e76309..3bd7f0ca7 100644 --- a/covalent_dispatcher/_db/upsert.py +++ b/covalent_dispatcher/_db/upsert.py @@ -50,9 +50,7 @@ ELECTRON_QELECTRON_DB_FILENAME = ELECTRON_FILENAMES["qelectron_db"] ELECTRON_ERROR_FILENAME = ELECTRON_FILENAMES["error"] ELECTRON_RESULTS_FILENAME = ELECTRON_FILENAMES["output"] -ELECTRON_DEPS_FILENAME = ELECTRON_FILENAMES["deps"] -ELECTRON_CALL_BEFORE_FILENAME = ELECTRON_FILENAMES["call_before"] -ELECTRON_CALL_AFTER_FILENAME = ELECTRON_FILENAMES["call_after"] +ELECTRON_HOOKS_FILENAME = ELECTRON_FILENAMES["hooks"] ELECTRON_STORAGE_TYPE = "file" LATTICE_FUNCTION_FILENAME = LATTICE_FILENAMES["workflow_function"] LATTICE_FUNCTION_STRING_FILENAME = LATTICE_FILENAMES["workflow_function_string"] @@ -62,9 +60,7 @@ LATTICE_NAMED_ARGS_FILENAME = LATTICE_FILENAMES["named_args"] LATTICE_NAMED_KWARGS_FILENAME = LATTICE_FILENAMES["named_kwargs"] LATTICE_RESULTS_FILENAME = LATTICE_FILENAMES["result"] -LATTICE_DEPS_FILENAME = LATTICE_FILENAMES["deps"] -LATTICE_CALL_BEFORE_FILENAME = LATTICE_FILENAMES["call_before"] -LATTICE_CALL_AFTER_FILENAME = LATTICE_FILENAMES["call_after"] +LATTICE_HOOKS_FILENAME = LATTICE_FILENAMES["hooks"] LATTICE_COVA_IMPORTS_FILENAME = LATTICE_FILENAMES["cova_imports"] LATTICE_LATTICE_IMPORTS_FILENAME = LATTICE_FILENAMES["lattice_imports"] LATTICE_STORAGE_TYPE = "file" @@ -115,9 +111,7 @@ def _lattice_data(session: Session, result: Result, electron_id: int = None) -> ("named_args", LATTICE_NAMED_ARGS_FILENAME, result.lattice.named_args), ("named_kwargs", LATTICE_NAMED_KWARGS_FILENAME, result.lattice.named_kwargs), ("result", LATTICE_RESULTS_FILENAME, result._result), - ("deps", LATTICE_DEPS_FILENAME, result.lattice.metadata["deps"]), - ("call_before", LATTICE_CALL_BEFORE_FILENAME, result.lattice.metadata["call_before"]), - ("call_after", LATTICE_CALL_AFTER_FILENAME, result.lattice.metadata["call_after"]), + ("hooks", LATTICE_HOOKS_FILENAME, result.lattice.metadata["hooks"]), ("cova_imports", LATTICE_COVA_IMPORTS_FILENAME, result.lattice.cova_imports), ("lattice_imports", LATTICE_LATTICE_IMPORTS_FILENAME, result.lattice.lattice_imports), ]: @@ -170,9 +164,7 @@ def _lattice_data(session: Session, result: Result, electron_id: int = None) -> "named_args_filename": LATTICE_NAMED_ARGS_FILENAME, "named_kwargs_filename": LATTICE_NAMED_KWARGS_FILENAME, "results_filename": LATTICE_RESULTS_FILENAME, - "deps_filename": LATTICE_DEPS_FILENAME, - "call_before_filename": LATTICE_CALL_BEFORE_FILENAME, - "call_after_filename": LATTICE_CALL_AFTER_FILENAME, + "hooks_filename": LATTICE_HOOKS_FILENAME, "cova_imports_filename": LATTICE_COVA_IMPORTS_FILENAME, "lattice_imports_filename": LATTICE_LATTICE_IMPORTS_FILENAME, "results_dir": results_dir, @@ -294,16 +286,10 @@ def _electron_data( ("function", ELECTRON_FUNCTION_FILENAME, tg.get_node_value(node_id, "function")), ("function_string", ELECTRON_FUNCTION_STRING_FILENAME, function_string), ("value", ELECTRON_VALUE_FILENAME, node_value), - ("deps", ELECTRON_DEPS_FILENAME, tg.get_node_value(node_id, "metadata")["deps"]), ( - "call_before", - ELECTRON_CALL_BEFORE_FILENAME, - tg.get_node_value(node_id, "metadata")["call_before"], - ), - ( - "call_after", - ELECTRON_CALL_AFTER_FILENAME, - tg.get_node_value(node_id, "metadata")["call_after"], + "hooks", + ELECTRON_HOOKS_FILENAME, + tg.get_node_value(node_id, "metadata")["hooks"], ), ("stdout", ELECTRON_STDOUT_FILENAME, node_stdout), ("stderr", ELECTRON_STDERR_FILENAME, node_stderr), @@ -359,9 +345,7 @@ def _electron_data( "stdout_filename": ELECTRON_STDOUT_FILENAME, "stderr_filename": ELECTRON_STDERR_FILENAME, "error_filename": ELECTRON_ERROR_FILENAME, - "deps_filename": ELECTRON_DEPS_FILENAME, - "call_before_filename": ELECTRON_CALL_BEFORE_FILENAME, - "call_after_filename": ELECTRON_CALL_AFTER_FILENAME, + "hooks_filename": ELECTRON_HOOKS_FILENAME, "qelectron_data_exists": node_qelectron_data_exists, "job_id": job_row.id, "created_at": timestamp, diff --git a/covalent_dispatcher/_db/write_result_to_db.py b/covalent_dispatcher/_db/write_result_to_db.py index a334e395b..9d928c1ec 100644 --- a/covalent_dispatcher/_db/write_result_to_db.py +++ b/covalent_dispatcher/_db/write_result_to_db.py @@ -98,9 +98,7 @@ def transaction_insert_lattices_data( named_args_filename: str, named_kwargs_filename: str, results_filename: str, - deps_filename: str, - call_before_filename: str, - call_after_filename: str, + hooks_filename: str, cova_imports_filename: str, lattice_imports_filename: str, results_dir: str, @@ -138,9 +136,7 @@ def transaction_insert_lattices_data( named_args_filename=named_args_filename, named_kwargs_filename=named_kwargs_filename, results_filename=results_filename, - deps_filename=deps_filename, - call_before_filename=call_before_filename, - call_after_filename=call_after_filename, + hooks_filename=hooks_filename, cova_imports_filename=cova_imports_filename, lattice_imports_filename=lattice_imports_filename, results_dir=results_dir, @@ -248,9 +244,7 @@ def transaction_insert_electrons_data( stdout_filename: str, stderr_filename: str, error_filename: str, - deps_filename: str, - call_before_filename: str, - call_after_filename: str, + hooks_filename: str, job_id: int, qelectron_data_exists: bool, created_at: dt, @@ -291,9 +285,7 @@ def transaction_insert_electrons_data( stdout_filename=stdout_filename, stderr_filename=stderr_filename, error_filename=error_filename, - deps_filename=deps_filename, - call_before_filename=call_before_filename, - call_after_filename=call_after_filename, + hooks_filename=hooks_filename, qelectron_data_exists=qelectron_data_exists, is_active=True, job_id=job_id, diff --git a/covalent_dispatcher/_service/models.py b/covalent_dispatcher/_service/models.py index 43ac3410a..2d2f7db10 100644 --- a/covalent_dispatcher/_service/models.py +++ b/covalent_dispatcher/_service/models.py @@ -24,45 +24,6 @@ from covalent._shared_files.schemas.result import ResultSchema -# # Copied from _dal -# RESULT_ASSET_KEYS = { -# "inputs", -# "result", -# "error", -# } - -# # Copied from _dal -# LATTICE_ASSET_KEYS = { -# "workflow_function", -# "workflow_function_string", -# "__doc__", -# "named_args", -# "named_kwargs", -# "cova_imports", -# "lattice_imports", -# # metadata -# "executor_data", -# "workflow_executor_data", -# "deps", -# "call_before", -# "call_after", -# } - -# # Copied from _dal -# ELECTRON_ASSET_KEYS = { -# "function", -# "function_string", -# "output", -# "value", -# "error", -# "stdout", -# "stderr", -# # electron metadata -# "deps", -# "call_before", -# "call_after", -# } - range_regex = "bytes=([0-9]+)-([0-9]*)" range_pattern = re.compile(range_regex) @@ -82,9 +43,7 @@ class LatticeAssetKey(str, Enum): inputs = "inputs" named_args = "named_args" named_kwargs = "named_kwargs" - deps = "deps" - call_before = "call_before" - call_after = "call_after" + hooks = "hooks" cova_imports = "cova_imports" lattice_imports = "lattice_imports" @@ -94,13 +53,11 @@ class ElectronAssetKey(str, Enum): function_string = "function_string" output = "output" value = "value" - deps = "deps" + hooks = "hooks" error = "error" stdout = "stdout" stderr = "stderr" qelectron_db = "qelectron_db" - call_before = "call_before" - call_after = "call_after" class ExportResponseSchema(BaseModel): diff --git a/covalent_migrations/versions/3727163f275c_consolidate_electron_hooks.py b/covalent_migrations/versions/3727163f275c_consolidate_electron_hooks.py new file mode 100644 index 000000000..d1954d3f7 --- /dev/null +++ b/covalent_migrations/versions/3727163f275c_consolidate_electron_hooks.py @@ -0,0 +1,73 @@ +# Copyright 2021 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Consolidate electron hooks + +Revision ID: 3727163f275c +Revises: 1142d81b29b8 +Create Date: 2023-12-07 10:47:38.837720 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +# pragma: allowlist nextline secret +revision = "3727163f275c" +# pragma: allowlist nextline secret +down_revision = "1142d81b29b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("electron_dependency", schema=None) as batch_op: + batch_op.create_foreign_key("child_electron_link", "electrons", ["electron_id"], ["id"]) + + with op.batch_alter_table("electrons", schema=None) as batch_op: + batch_op.add_column(sa.Column("hooks_filename", sa.Text(), nullable=True)) + batch_op.drop_column("call_after_filename") + batch_op.drop_column("call_before_filename") + batch_op.drop_column("deps_filename") + + with op.batch_alter_table("lattices", schema=None) as batch_op: + batch_op.add_column(sa.Column("hooks_filename", sa.Text(), nullable=True)) + batch_op.drop_column("call_after_filename") + batch_op.drop_column("call_before_filename") + batch_op.drop_column("deps_filename") + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("lattices", schema=None) as batch_op: + batch_op.add_column(sa.Column("deps_filename", sa.TEXT(), nullable=True)) + batch_op.add_column(sa.Column("call_before_filename", sa.TEXT(), nullable=True)) + batch_op.add_column(sa.Column("call_after_filename", sa.TEXT(), nullable=True)) + batch_op.drop_column("hooks_filename") + + with op.batch_alter_table("electrons", schema=None) as batch_op: + batch_op.add_column(sa.Column("deps_filename", sa.TEXT(), nullable=True)) + batch_op.add_column(sa.Column("call_before_filename", sa.TEXT(), nullable=True)) + batch_op.add_column(sa.Column("call_after_filename", sa.TEXT(), nullable=True)) + batch_op.drop_column("hooks_filename") + + with op.batch_alter_table("electron_dependency", schema=None) as batch_op: + batch_op.drop_constraint("child_electron_link", type_="foreignkey") + + # ### end Alembic commands ### diff --git a/covalent_ui/api/v1/data_layer/electron_dal.py b/covalent_ui/api/v1/data_layer/electron_dal.py index 4a4707146..5f85c965a 100644 --- a/covalent_ui/api/v1/data_layer/electron_dal.py +++ b/covalent_ui/api/v1/data_layer/electron_dal.py @@ -258,9 +258,7 @@ def get_electrons_id(self, dispatch_id, electron_id) -> Electron: Electron.results_filename, Electron.value_filename, Electron.stdout_filename, - Electron.deps_filename, - Electron.call_before_filename, - Electron.call_after_filename, + Electron.hooks_filename, Electron.stderr_filename, Electron.error_filename, Electron.name, diff --git a/covalent_ui/api/v1/database/schema/electron.py b/covalent_ui/api/v1/database/schema/electron.py index 635240918..01a36bca8 100644 --- a/covalent_ui/api/v1/database/schema/electron.py +++ b/covalent_ui/api/v1/database/schema/electron.py @@ -44,9 +44,7 @@ class Electron(Base): key: For generated and subscript nodes stdout_filename: Name of the file containing standard output generated by the task stderr_filename: Name of the file containing standard error generated by the task - deps_filename: Name of the file containing depends instances of DepsBash and DepsPip - call_before_filename: Name of the file containing list of DepsCall objects - call_after_filename : Name of the file containing list of DepsCall objects + hooks_filename: Name of the file containing depends instances of DepsBash, DepsPip, and DepsCall error_filename: Name of the file containing execution information generated at runtime is_active: Status of the record, 1: active and 0: inactive job_id: ID for circuit_info @@ -106,13 +104,7 @@ class Electron(Base): stdout_filename = Column(Text) # Name of the file containing the electron execution dependencies - deps_filename = Column(Text) - - # Name of the file containing the functions that are called before electron execution - call_before_filename = Column(Text) - - # Name of the file containing the functions that are called after electron execution - call_after_filename = Column(Text) + hooks_filename = Column(Text) # Name of the file containing standard error generated by the task stderr_filename = Column(Text) diff --git a/covalent_ui/api/v1/database/schema/lattices.py b/covalent_ui/api/v1/database/schema/lattices.py index 97212a9f1..330fa4915 100644 --- a/covalent_ui/api/v1/database/schema/lattices.py +++ b/covalent_ui/api/v1/database/schema/lattices.py @@ -107,13 +107,7 @@ class Lattice(Base): results_filename = Column(Text) # Name of the file containing the default electron dependencies - deps_filename = Column(Text) - - # Name of the file containing the default list of callables before electrons are executed - call_before_filename = Column(Text) - - # Name of the file containing the default list of callables after electrons are executed - call_after_filename = Column(Text) + hooks_filename = Column(Text) # Name of the file containing the set of cova imports cova_imports_filename = Column(Text) diff --git a/covalent_ui/api/v1/models/electrons_model.py b/covalent_ui/api/v1/models/electrons_model.py index 4080a30ae..cf94f355e 100644 --- a/covalent_ui/api/v1/models/electrons_model.py +++ b/covalent_ui/api/v1/models/electrons_model.py @@ -117,9 +117,7 @@ class ElectronFileOutput(str, Enum): RESULT = "result" VALUE = "value" STDOUT = "stdout" - DEPS = "deps" - CALL_BEFORE = "call_before" - CALL_AFTER = "call_after" + HOOKS = "hooks" ERROR = "error" INFO = "info" INPUTS = "inputs" diff --git a/covalent_ui/api/v1/routes/end_points/electron_routes.py b/covalent_ui/api/v1/routes/end_points/electron_routes.py index edd074dc5..d58a93df2 100644 --- a/covalent_ui/api/v1/routes/end_points/electron_routes.py +++ b/covalent_ui/api/v1/routes/end_points/electron_routes.py @@ -175,7 +175,7 @@ def get_electron_file(dispatch_id: uuid.UUID, electron_id: int, name: ElectronFi dispatch_id: Dispatch id of lattice/sublattice electron_id: Transport graph node id of a electron name: refers file type, like inputs, function_string, function, executor, result, value, key, - stdout, deps, call_before, call_after, error, info + stdout, hooks, error, info Returns: Returns electron details based on the given name """ @@ -221,14 +221,8 @@ def get_electron_file(dispatch_id: uuid.UUID, electron_id: int, name: ElectronFi elif name == "stdout": response = handler.read_from_text(result["stdout_filename"]) return ElectronFileResponse(data=response) - elif name == "deps": - response = handler.read_from_serialized(result["deps_filename"]) - return ElectronFileResponse(data=response) - elif name == "call_before": - response = handler.read_from_serialized(result["call_before_filename"]) - return ElectronFileResponse(data=response) - elif name == "call_after": - response = handler.read_from_serialized(result["call_after_filename"]) + elif name == "hooks": + response = handler.read_from_serialized(result["hooks_filename"]) return ElectronFileResponse(data=response) elif name == "error": # Error and stderr won't be both populated if `error` diff --git a/tests/covalent_dispatcher_tests/_core/runner_ng_test.py b/tests/covalent_dispatcher_tests/_core/runner_ng_test.py index c1d095cdb..4e6f8a88d 100644 --- a/tests/covalent_dispatcher_tests/_core/runner_ng_test.py +++ b/tests/covalent_dispatcher_tests/_core/runner_ng_test.py @@ -158,14 +158,10 @@ async def test_submit_abstract_task_group(mocker, task_cancelled): } mock_function_uri_0 = me.get_upload_uri(task_group_metadata, "function-0") - mock_deps_uri_0 = me.get_upload_uri(task_group_metadata, "deps-0") - mock_cb_uri_0 = me.get_upload_uri(task_group_metadata, "call_before-0") - mock_ca_uri_0 = me.get_upload_uri(task_group_metadata, "call_after-0") + mock_hooks_uri_0 = me.get_upload_uri(task_group_metadata, "hooks-0") mock_function_uri_3 = me.get_upload_uri(task_group_metadata, "function-3") - mock_deps_uri_3 = me.get_upload_uri(task_group_metadata, "deps-3") - mock_cb_uri_3 = me.get_upload_uri(task_group_metadata, "call_before-3") - mock_ca_uri_3 = me.get_upload_uri(task_group_metadata, "call_after-3") + mock_hooks_uri_3 = me.get_upload_uri(task_group_metadata, "hooks-3") mock_node_upload_uri_1 = me.get_upload_uri(task_group_metadata, "node_1") mock_node_upload_uri_2 = me.get_upload_uri(task_group_metadata, "node_2") @@ -173,14 +169,8 @@ async def test_submit_abstract_task_group(mocker, task_cancelled): mock_function_id_0 = 0 mock_args_ids = abstract_inputs["args"] mock_kwargs_ids = abstract_inputs["kwargs"] - mock_deps_id_0 = "deps-0" - mock_cb_id_0 = "call_before-0" - mock_ca_id_0 = "call_after-0" mock_function_id_3 = 3 - mock_deps_id_3 = "deps-3" - mock_cb_id_3 = "call_before-3" - mock_ca_id_3 = "call_after-3" resources = { "functions": { @@ -191,32 +181,19 @@ async def test_submit_abstract_task_group(mocker, task_cancelled): 1: mock_node_upload_uri_1, 2: mock_node_upload_uri_2, }, - "deps": { - mock_deps_id_0: mock_deps_uri_0, - mock_cb_id_0: mock_cb_uri_0, - mock_ca_id_0: mock_ca_uri_0, - mock_deps_id_3: mock_deps_uri_3, - mock_cb_id_3: mock_cb_uri_3, - mock_ca_id_3: mock_ca_uri_3, - }, + "hooks": {0: mock_hooks_uri_0, 3: mock_hooks_uri_3}, } mock_task_spec_0 = { "function_id": mock_function_id_0, "args_ids": mock_args_ids, "kwargs_ids": mock_kwargs_ids, - "deps_id": mock_deps_id_0, - "call_before_id": mock_cb_id_0, - "call_after_id": mock_ca_id_0, } mock_task_spec_3 = { "function_id": mock_function_id_3, "args_ids": mock_args_ids, "kwargs_ids": mock_kwargs_ids, - "deps_id": mock_deps_id_3, - "call_before_id": mock_cb_id_3, - "call_after_id": mock_ca_id_3, } mock_task_0 = { diff --git a/tests/covalent_dispatcher_tests/_db/update_test.py b/tests/covalent_dispatcher_tests/_db/update_test.py index b02875e0c..567c83bc9 100644 --- a/tests/covalent_dispatcher_tests/_db/update_test.py +++ b/tests/covalent_dispatcher_tests/_db/update_test.py @@ -128,9 +128,7 @@ def test_result_persist_workflow_1(test_db, result_1, mocker): assert lattice_row.electron_id is None assert lattice_row.executor == "local" assert lattice_row.workflow_executor == "local" - assert lattice_row.deps_filename == upsert.LATTICE_DEPS_FILENAME - assert lattice_row.call_before_filename == upsert.LATTICE_CALL_BEFORE_FILENAME - assert lattice_row.call_after_filename == upsert.LATTICE_CALL_AFTER_FILENAME + assert lattice_row.hooks_filename == upsert.LATTICE_HOOKS_FILENAME assert lattice_row.root_dispatch_id == "dispatch_1" assert lattice_row.results_dir == result_1.results_dir @@ -179,22 +177,10 @@ def test_result_persist_workflow_1(test_db, result_1, mocker): if electron.transport_graph_node_id == 1: assert ( local_store.load_file( - storage_path=electron.storage_path, filename=electron.deps_filename + storage_path=electron.storage_path, filename=electron.hooks_filename ) == {} ) - assert ( - local_store.load_file( - storage_path=electron.storage_path, filename=electron.call_before_filename - ) - == [] - ) - assert ( - local_store.load_file( - storage_path=electron.storage_path, filename=electron.call_after_filename - ) - == [] - ) if electron.transport_graph_node_id == 3: executor_data = json.loads(electron.executor_data) diff --git a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py index e5558fc05..26b114913 100644 --- a/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py +++ b/tests/covalent_dispatcher_tests/_db/write_result_to_db_test.py @@ -67,9 +67,7 @@ STDERR_FILENAME = "stderr.log" ERROR_FILENAME = "error.log" TRANSPORT_GRAPH_FILENAME = "transport_graph.pkl" -DEPS_FILENAME = "deps.pkl" -CALL_BEFORE_FILENAME = "call_before.pkl" -CALL_AFTER_FILENAME = "call_after.pkl" +HOOKS_FILENAME = "hooks.pkl" COVA_IMPORTS_FILENAME = "cova_imports.json" LATTICE_IMPORTS_FILENAME = "lattice_imports.txt" RESULTS_DIR = "/tmp/results" @@ -131,9 +129,7 @@ def get_lattice_kwargs( named_args_filename=NAMED_ARGS_FILENAME, named_kwargs_filename=NAMED_KWARGS_FILENAME, results_filename=RESULTS_FILENAME, - deps_filename=DEPS_FILENAME, - call_before_filename=CALL_BEFORE_FILENAME, - call_after_filename=CALL_AFTER_FILENAME, + hooks_filename=HOOKS_FILENAME, cova_imports_filename=COVA_IMPORTS_FILENAME, lattice_imports_filename=LATTICE_IMPORTS_FILENAME, results_dir=RESULTS_DIR, @@ -166,9 +162,7 @@ def get_lattice_kwargs( "named_args_filename": named_args_filename, "named_kwargs_filename": named_kwargs_filename, "results_filename": results_filename, - "deps_filename": deps_filename, - "call_before_filename": call_before_filename, - "call_after_filename": call_after_filename, + "hooks_filename": hooks_filename, "cova_imports_filename": cova_imports_filename, "lattice_imports_filename": lattice_imports_filename, "results_dir": results_dir, @@ -198,9 +192,7 @@ def get_electron_kwargs( stdout_filename=STDOUT_FILENAME, stderr_filename=STDERR_FILENAME, error_filename=ERROR_FILENAME, - deps_filename=DEPS_FILENAME, - call_before_filename=CALL_BEFORE_FILENAME, - call_after_filename=CALL_AFTER_FILENAME, + hooks_filename=HOOKS_FILENAME, job_id=1, qelectron_data_exists=False, created_at=None, @@ -228,9 +220,7 @@ def get_electron_kwargs( "stdout_filename": stdout_filename, "stderr_filename": stderr_filename, "error_filename": error_filename, - "deps_filename": deps_filename, - "call_before_filename": call_before_filename, - "call_after_filename": call_after_filename, + "hooks_filename": hooks_filename, "job_id": job_id, "qelectron_data_exists": qelectron_data_exists, "created_at": created_at, @@ -299,9 +289,7 @@ def test_insert_lattices_data(test_db, mocker): assert lattice.named_args_filename == NAMED_ARGS_FILENAME assert lattice.named_kwargs_filename == NAMED_KWARGS_FILENAME assert lattice.results_filename == RESULTS_FILENAME - assert lattice.deps_filename == DEPS_FILENAME - assert lattice.call_before_filename == CALL_BEFORE_FILENAME - assert lattice.call_after_filename == CALL_AFTER_FILENAME + assert lattice.hooks_filename == HOOKS_FILENAME assert lattice.cova_imports_filename == COVA_IMPORTS_FILENAME assert lattice.lattice_imports_filename == LATTICE_IMPORTS_FILENAME assert lattice.results_dir == RESULTS_DIR diff --git a/tests/covalent_tests/executor/executor_plugins/dask_test.py b/tests/covalent_tests/executor/executor_plugins/dask_test.py index 548c35cf0..39dc80775 100644 --- a/tests/covalent_tests/executor/executor_plugins/dask_test.py +++ b/tests/covalent_tests/executor/executor_plugins/dask_test.py @@ -38,7 +38,7 @@ ResourceMap, TaskSpec, dask_wrapper, - run_task_from_uris_alt, + run_task_group_alt, ) from covalent.executor.utils.serialize import serialize_node_asset @@ -292,11 +292,14 @@ def task(x, y): deps = {} call_before = [] call_after = [] + hooks = { + "deps": deps, + "call_before": call_before, + "call_after": call_after, + } ser_task = serialize_node_asset(TransportableObject(task), "function") - ser_deps = serialize_node_asset(deps, "deps") - ser_cb = serialize_node_asset(deps, "call_before") - ser_ca = serialize_node_asset(deps, "call_after") + ser_hooks = serialize_node_asset(deps, "hooks") ser_x = serialize_node_asset(x, "output") ser_y = serialize_node_asset(y, "output") @@ -304,17 +307,9 @@ def task(x, y): node_0_file.write(ser_task) node_0_file.flush() - deps_file = tempfile.NamedTemporaryFile("wb") - deps_file.write(ser_deps) - deps_file.flush() - - cb_file = tempfile.NamedTemporaryFile("wb") - cb_file.write(ser_cb) - cb_file.flush() - - ca_file = tempfile.NamedTemporaryFile("wb") - ca_file.write(ser_ca) - ca_file.flush() + hooks_file = tempfile.NamedTemporaryFile("wb") + hooks_file.write(ser_hooks) + hooks_file.flush() node_1_file = tempfile.NamedTemporaryFile("wb") node_1_file.write(ser_x) @@ -328,9 +323,6 @@ def task(x, y): function_id=0, args_ids=[1, 2], kwargs_ids={}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resources = ResourceMap( @@ -341,10 +333,8 @@ def task(x, y): 1: node_1_file.name, 2: node_2_file.name, }, - deps={ - "deps": deps_file.name, - "call_before": cb_file.name, - "call_after": ca_file.name, + hooks={ + 0: hooks_file.name, }, ) @@ -379,7 +369,7 @@ async def run_dask_job(task_specs, resources, task_group_metadata): assert task_update.assets["output"].size == os.path.getsize(output_uri) -def test_run_task_from_uris_alt(): +def test_run_task_group_alt(): """Test the wrapper submitted to dask""" def task(x, y): @@ -399,10 +389,14 @@ def task(x, y): call_before = [ct.DepsBash([f"echo Hello > {cb_tmpfile.name}"]).to_dict()] call_after = [ct.DepsBash(f"echo Bye > {ca_tmpfile.name}").to_dict()] + hooks = { + "deps": deps, + "call_before": call_before, + "call_after": call_after, + } + ser_task = serialize_node_asset(TransportableObject(task), "function") - ser_deps = serialize_node_asset(deps, "deps") - ser_cb = serialize_node_asset(call_before, "call_before") - ser_ca = serialize_node_asset(call_after, "call_after") + ser_hooks = serialize_node_asset(hooks, "hooks") ser_x = serialize_node_asset(x, "output") ser_y = serialize_node_asset(y, "output") @@ -410,17 +404,9 @@ def task(x, y): node_0_file.write(ser_task) node_0_file.flush() - deps_file = tempfile.NamedTemporaryFile("wb") - deps_file.write(ser_deps) - deps_file.flush() - - cb_file = tempfile.NamedTemporaryFile("wb") - cb_file.write(ser_cb) - cb_file.flush() - - ca_file = tempfile.NamedTemporaryFile("wb") - ca_file.write(ser_ca) - ca_file.flush() + hooks_file = tempfile.NamedTemporaryFile("wb") + hooks_file.write(ser_hooks) + hooks_file.flush() node_1_file = tempfile.NamedTemporaryFile("wb") node_1_file.write(ser_x) @@ -434,9 +420,6 @@ def task(x, y): function_id=0, args_ids=[1, 2], kwargs_ids={}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resources = ResourceMap( @@ -447,10 +430,8 @@ def task(x, y): 1: node_1_file.name, 2: node_2_file.name, }, - deps={ - "deps": deps_file.name, - "call_before": cb_file.name, - "call_after": ca_file.name, + hooks={ + 0: hooks_file.name, }, ) @@ -467,7 +448,7 @@ def task(x, y): results_dir = tempfile.TemporaryDirectory() - run_task_from_uris_alt( + run_task_group_alt( task_specs=[task_spec.dict()], resources=resources.dict(), output_uris=[ @@ -489,7 +470,7 @@ def task(x, y): assert f.read() == "Bye\n" -def test_run_task_from_uris_alt_exception(): +def test_run_task_group_alt_exception(): """Test the wrapper submitted to dask""" def task(x, y): @@ -509,10 +490,14 @@ def task(x, y): call_before = [ct.DepsBash([f"echo Hello > {cb_tmpfile.name}"]).to_dict()] call_after = [ct.DepsBash(f"echo Bye > {ca_tmpfile.name}").to_dict()] + hooks = { + "deps": deps, + "call_before": call_before, + "call_after": call_after, + } + ser_task = serialize_node_asset(TransportableObject(task), "function") - ser_deps = serialize_node_asset(deps, "deps") - ser_cb = serialize_node_asset(call_before, "call_before") - ser_ca = serialize_node_asset(call_after, "call_after") + ser_hooks = serialize_node_asset(hooks, "hooks") ser_x = serialize_node_asset(x, "output") ser_y = serialize_node_asset(y, "output") @@ -520,17 +505,9 @@ def task(x, y): node_0_file.write(ser_task) node_0_file.flush() - deps_file = tempfile.NamedTemporaryFile("wb") - deps_file.write(ser_deps) - deps_file.flush() - - cb_file = tempfile.NamedTemporaryFile("wb") - cb_file.write(ser_cb) - cb_file.flush() - - ca_file = tempfile.NamedTemporaryFile("wb") - ca_file.write(ser_ca) - ca_file.flush() + hooks_file = tempfile.NamedTemporaryFile("wb") + hooks_file.write(ser_hooks) + hooks_file.flush() node_1_file = tempfile.NamedTemporaryFile("wb") node_1_file.write(ser_x) @@ -544,9 +521,6 @@ def task(x, y): function_id=0, args_ids=[1], kwargs_ids={"y": 2}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resources = ResourceMap( @@ -557,10 +531,8 @@ def task(x, y): 1: f"file://{node_1_file.name}", 2: f"file://{node_2_file.name}", }, - deps={ - "deps": f"file://{deps_file.name}", - "call_before": f"file://{cb_file.name}", - "call_after": f"file://{ca_file.name}", + hooks={ + 0: f"file://{hooks_file.name}", }, ) @@ -577,7 +549,7 @@ def task(x, y): results_dir = tempfile.TemporaryDirectory() - run_task_from_uris_alt( + run_task_group_alt( task_specs=[task_spec.model_dump()], resources=resources.model_dump(), output_uris=[ diff --git a/tests/covalent_tests/executor/executor_plugins/local_test.py b/tests/covalent_tests/executor/executor_plugins/local_test.py index 331e44579..e183497f5 100644 --- a/tests/covalent_tests/executor/executor_plugins/local_test.py +++ b/tests/covalent_tests/executor/executor_plugins/local_test.py @@ -35,7 +35,7 @@ LocalExecutor, StatusEnum, TaskSpec, - run_task_from_uris, + run_task_group, ) from covalent.executor.schemas import ResourceMap from covalent.executor.utils.serialize import serialize_node_asset @@ -239,7 +239,7 @@ def test_local_executor_get_cancel_requested(mocker): assert mock_app_log.call_count == 2 -def test_run_task_from_uris(mocker): +def test_run_task_group(mocker): """Test the wrapper submitted to local""" def task(x, y): @@ -264,11 +264,14 @@ def task(x, y): call_before = [] call_after = [] + hooks = { + "deps": deps, + "call_before": call_before, + "call_after": call_after, + } ser_task = serialize_node_asset(TransportableObject(task), "function") - ser_deps = serialize_node_asset(deps, "deps") - ser_cb = serialize_node_asset(call_before, "call_before") - ser_ca = serialize_node_asset(call_after, "call_after") + ser_hooks = serialize_node_asset(hooks, "hooks") ser_x = serialize_node_asset(x, "output") ser_y = serialize_node_asset(y, "output") @@ -279,20 +282,10 @@ def task(x, y): f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/function" ) - deps_file = tempfile.NamedTemporaryFile("wb") - deps_file.write(ser_deps) - deps_file.flush() - deps_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/deps" - - cb_file = tempfile.NamedTemporaryFile("wb") - cb_file.write(ser_cb) - cb_file.flush() - cb_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/call_before" - - ca_file = tempfile.NamedTemporaryFile("wb") - ca_file.write(ser_ca) - ca_file.flush() - ca_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/call_after" + hooks_file = tempfile.NamedTemporaryFile("wb") + hooks_file.write(ser_hooks) + hooks_file.flush() + hooks_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/hooks" node_1_file = tempfile.NamedTemporaryFile("wb") node_1_file.write(ser_x) @@ -308,18 +301,13 @@ def task(x, y): function_id=0, args_ids=[1, 2], kwargs_ids={}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resources = { node_0_function_url: ser_task, node_1_output_url: ser_x, node_2_output_url: ser_y, - deps_url: ser_deps, - cb_url: ser_cb, - ca_url: ser_ca, + hooks_url: ser_hooks, } def mock_req_get(url, stream): @@ -347,9 +335,8 @@ def mock_req_post(url, files): results_dir = tempfile.TemporaryDirectory() - run_task_from_uris( + run_task_group( task_specs=[task_spec.dict()], - resources={}, output_uris=[ (result_file.name, stdout_file.name, stderr_file.name, qelectron_db_file.name) ], @@ -371,7 +358,7 @@ def mock_req_post(url, files): mock_put.assert_called() -def test_run_task_from_uris_exception(mocker): +def test_run_task_group_exception(mocker): """Test the wrapper submitted to local""" def task(x, y): @@ -397,10 +384,14 @@ def task(x, y): call_before = [] call_after = [] + hooks = { + "deps": deps, + "call_before": call_before, + "call_after": call_after, + } + ser_task = serialize_node_asset(TransportableObject(task), "function") - ser_deps = serialize_node_asset(deps, "deps") - ser_cb = serialize_node_asset(call_before, "call_before") - ser_ca = serialize_node_asset(call_after, "call_after") + ser_hooks = serialize_node_asset(hooks, "hooks") ser_x = serialize_node_asset(x, "output") ser_y = serialize_node_asset(y, "output") @@ -411,20 +402,10 @@ def task(x, y): f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/function" ) - deps_file = tempfile.NamedTemporaryFile("wb") - deps_file.write(ser_deps) - deps_file.flush() - deps_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/deps" - - cb_file = tempfile.NamedTemporaryFile("wb") - cb_file.write(ser_cb) - cb_file.flush() - cb_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/call_before" - - ca_file = tempfile.NamedTemporaryFile("wb") - ca_file.write(ser_ca) - ca_file.flush() - ca_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/call_after" + hooks_file = tempfile.NamedTemporaryFile("wb") + hooks_file.write(ser_hooks) + hooks_file.flush() + hooks_url = f"{server_url}/api/v2/dispatches/{dispatch_id}/electrons/0/assets/hooks" node_1_file = tempfile.NamedTemporaryFile("wb") node_1_file.write(ser_x) @@ -440,18 +421,13 @@ def task(x, y): function_id=0, args_ids=[1], kwargs_ids={"y": 2}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resources = { node_0_function_url: ser_task, node_1_output_url: ser_x, node_2_output_url: ser_y, - deps_url: ser_deps, - cb_url: ser_cb, - ca_url: ser_ca, + hooks_url: ser_hooks, } def mock_req_get(url, stream): @@ -479,9 +455,8 @@ def mock_req_post(url, files): results_dir = tempfile.TemporaryDirectory() - run_task_from_uris( + run_task_group( task_specs=[task_spec.dict()], - resources={}, output_uris=[ (result_file.name, stdout_file.name, stderr_file.name, qelectron_db_file.name) ], @@ -538,15 +513,12 @@ def mock_proc_pool_submit(mock_future): function_id=0, args_ids=[1], kwargs_ids={"y": 2}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) ], "resources": ResourceMap( functions={0: "mock_function_uri"}, inputs={1: "mock_input_uri"}, - deps={"deps": "mock_deps_uri"}, + hooks={0: "mock_hooks_uri"}, ), "task_group_metadata": {"dispatch_id": "1", "node_ids": ["1"], "task_group_id": "1"}, "expected_output_uris": [("mock_path", "mock_path", "mock_path", "mock_path")], @@ -560,15 +532,12 @@ def mock_proc_pool_submit(mock_future): function_id=0, args_ids=[1], kwargs_ids={"y": 2}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) ], "resources": ResourceMap( functions={0: "mock_function_uri"}, inputs={1: "mock_input_uri"}, - deps={"deps": "mock_deps_uri"}, + hooks={0: "mock_hooks_uri"}, ), "task_group_metadata": {"dispatch_id": "1", "node_ids": ["1"], "task_group_id": "1"}, "expected_output_uris": [("mock_path", "mock_path", "mock_path", "mock_path")], @@ -605,9 +574,8 @@ def test_send_internal( mock_os_path_join.assert_called() mock_format_server_url.assert_called_once_with() mock_proc_pool_submit.assert_called_once_with( - run_task_from_uris, + run_task_group, list(map(lambda t: t.dict(), test_case["task_specs"])), - test_case["resources"].dict(), test_case["expected_output_uris"], "mock_cache_dir", test_case["task_group_metadata"], @@ -627,14 +595,11 @@ async def test_send(mocker): function_id=0, args_ids=[1], kwargs_ids={"y": 2}, - deps_id="deps", - call_before_id="call_before", - call_after_id="call_after", ) resource = ResourceMap( functions={0: "mock_function_uri"}, inputs={1: "mock_input_uri"}, - deps={"deps": "mock_deps_uri"}, + hooks={0: "mock_hooks_uri"}, ) mock_loop = mocker.Mock() diff --git a/tests/covalent_tests/workflow/electron_metadata_test.py b/tests/covalent_tests/workflow/electron_metadata_test.py index a2b45f1a8..8971f91a4 100644 --- a/tests/covalent_tests/workflow/electron_metadata_test.py +++ b/tests/covalent_tests/workflow/electron_metadata_test.py @@ -54,9 +54,7 @@ def hello_world(x): assert metadata["executor"] == get_default_executor() elif "parameter" not in node_name: assert metadata["executor"] == "electron_executor" - assert metadata["deps"]["bash"] == electron_bash_dep.to_dict() - assert len(metadata["call_before"]) == 2 - assert len(metadata["call_after"]) == 0 + assert metadata["hooks"]["deps"]["bash"] == electron_bash_dep.to_dict() def test_electrons_get_lattice_metadata_2(): @@ -65,12 +63,9 @@ def test_electrons_get_lattice_metadata_2(): electron_bash_dep = ct.DepsBash(["yum install rustc"]) lattice_bash_dep = ct.DepsBash(["yum install kernel"]) - ft_after = ct.fs.FileTransfer( - "/home/ubuntu/src_file", "/home/ubuntu/dest_file", order=ct.fs.Order.AFTER - ) # Construct tasks as "electrons" - @ct.electron(files=[ft_after]) + @ct.electron def task(x): return x @@ -89,9 +84,7 @@ def hello_world(x): tg.deserialize_from_json(data) metadata = tg.get_node_value(0, "metadata") assert metadata["executor"] == "lattice_executor" - assert metadata["deps"]["bash"] == lattice_bash_dep.to_dict() - assert len(metadata["call_before"]) == 1 - assert len(metadata["call_after"]) == 1 + assert metadata["hooks"]["deps"]["bash"] == lattice_bash_dep.to_dict() def test_electrons_get_lattice_metadata_3(): diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index 941934f65..25bba748e 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -94,9 +94,11 @@ def workflow(x): "executor_data": {}, "workflow_executor": "my_postprocessor", "workflow_executor_data": {}, - "deps": {"bash": None, "pip": None}, - "call_before": [], - "call_after": [], + "hooks": { + "deps": {"bash": None, "pip": None}, + "call_before": [], + "call_after": [], + }, "triggers": "mock-trigger", "qelectron_data_exists": False, "results_dir": None, @@ -158,9 +160,11 @@ def workflow(x): "executor_data": {}, "workflow_executor": "my_postprocessor", "workflow_executor_data": {}, - "deps": {"bash": None, "pip": None}, - "call_before": [], - "call_after": [], + "hooks": { + "deps": {"bash": None, "pip": None}, + "call_before": [], + "call_after": [], + }, "triggers": "mock-trigger", "qelectron_data_exists": False, "results_dir": None, @@ -311,8 +315,8 @@ def workflow(x): task_metadata = workflow.transport_graph.get_node_value(0, "metadata") e_list_metadata = workflow.transport_graph.get_node_value(1, "metadata") - assert not list(e_list_metadata["call_before"]) - assert not list(e_list_metadata["call_after"]) + assert "call_before" not in e_list_metadata["hooks"] + assert "call_after" not in e_list_metadata["hooks"] assert e_list_metadata["executor"] == task_metadata["executor"] @@ -446,7 +450,7 @@ def sublattice(x): bound_electron = sublattice() assert bound_electron.metadata["executor"] == "dask" assert bound_electron.metadata["executor_data"] == {} - assert bound_electron.metadata["deps"]["bash"] == bash_dep.to_dict() + assert bound_electron.metadata["hooks"]["deps"]["bash"] == bash_dep.to_dict() for _, node_data in mock_workflow.transport_graph._graph.nodes(data=True): if node_data["name"].startswith(sublattice_prefix): assert "mock_task" in node_data["function_string"] diff --git a/tests/covalent_tests/workflow/lepton_test.py b/tests/covalent_tests/workflow/lepton_test.py index ad8f544a8..c0accbfa8 100644 --- a/tests/covalent_tests/workflow/lepton_test.py +++ b/tests/covalent_tests/workflow/lepton_test.py @@ -131,7 +131,7 @@ def test_lepton_init( electron_init_mock.assert_called_once_with("wrapper function") wrap_mock.assert_called_once_with() - assert set_metadata_mock.call_count == 6 + assert set_metadata_mock.call_count == 4 assert lepton.language == language assert lepton.function_name == function_name @@ -179,8 +179,8 @@ def test_local_file_transfer_support(is_from_file_remote, is_to_file_remote, ord mock_file_transfer = FileTransfer(from_file=mock_from_file, to_file=mock_to_file, order=order) mock_lepton_with_files = Lepton("python", command="mockcmd", files=[mock_file_transfer]) - call_before_deps = mock_lepton_with_files.get_metadata("call_before") - call_after_deps = mock_lepton_with_files.get_metadata("call_after") + call_before_deps = mock_lepton_with_files.get_metadata("hooks")["call_before"] + call_after_deps = mock_lepton_with_files.get_metadata("hooks")["call_after"] pre_hook_call_dep, file_transfer_call_dep = mock_file_transfer.cp() @@ -236,11 +236,11 @@ def test_http_file_transfer(order): files=[FileTransfer(from_file=mock_to_file, to_file=mock_from_file, strategy=HTTP())], ) - deps = mock_lepton_with_files.get_metadata("deps") - assert deps["bash"]["attributes"]["commands"] == [] + deps = mock_lepton_with_files.get_metadata("hooks")["deps"] + assert deps.get("bash") is None assert deps.get("pip") is None - call_before = mock_lepton_with_files.get_metadata("call_before") - call_after = mock_lepton_with_files.get_metadata("call_after") + call_before = mock_lepton_with_files.get_metadata("hooks")["call_before"] + call_after = mock_lepton_with_files.get_metadata("hooks")["call_after"] pre_hook_call_dep, file_transfer_call_dep = mock_file_download.cp() diff --git a/tests/covalent_tests/workflow/transport_test.py b/tests/covalent_tests/workflow/transport_test.py index 752c2db96..4c9ce6855 100644 --- a/tests/covalent_tests/workflow/transport_test.py +++ b/tests/covalent_tests/workflow/transport_test.py @@ -425,11 +425,15 @@ def test_encode_metadata(): le = LocalExecutor() bt = BaseTrigger() - metadata = {"executor": le, "workflow_executor": "local", "deps": {}} - metadata["deps"]["bash"] = ct.DepsBash("yum install gcc") - metadata["deps"]["pip"] = ct.DepsPip(["sklearn"]) - metadata["call_before"] = [] - metadata["call_after"] = [] + hooks = { + "deps": { + "bash": ct.DepsBash("yum install gcc"), + "pip": ct.DepsPip(["sklearn"]), + }, + "call_before": [], + "call_after": [], + } + metadata = {"executor": le, "workflow_executor": "local", "hooks": hooks} metadata["triggers"] = [bt] json_metadata = json.dumps(encode_metadata(metadata)) @@ -442,8 +446,8 @@ def test_encode_metadata(): assert new_metadata["workflow_executor_data"] == {} assert new_metadata["triggers"] == [bt.to_dict()] - assert ct.DepsBash("yum install gcc").to_dict() == new_metadata["deps"]["bash"] - assert ct.DepsPip(["sklearn"]).to_dict() == new_metadata["deps"]["pip"] + assert ct.DepsBash("yum install gcc").to_dict() == new_metadata["hooks"]["deps"]["bash"] + assert ct.DepsPip(["sklearn"]).to_dict() == new_metadata["hooks"]["deps"]["pip"] # Check idempotence assert encode_metadata(metadata) == encode_metadata(encode_metadata(metadata)) diff --git a/tests/covalent_ui_backend_tests/end_points/electrons_test.py b/tests/covalent_ui_backend_tests/end_points/electrons_test.py index 5b4b2ec90..e78c3ddd5 100644 --- a/tests/covalent_ui_backend_tests/end_points/electrons_test.py +++ b/tests/covalent_ui_backend_tests/end_points/electrons_test.py @@ -166,37 +166,9 @@ def test_electrons_details_stdout(): assert response.json() == test_data["response_data"] -def test_electrons_details_deps(): - """Test electrons for deps details""" - test_data = output_data["test_electrons_details"]["case_deps_1"] - response = object_test_template( - api_path=output_data["test_electrons_details"]["api_path"], - app=fastapi_app, - method_type=MethodType.GET, - path=test_data["path"], - ) - assert response.status_code == test_data["status_code"] - if "response_data" in test_data: - assert response.json() == test_data["response_data"] - - -def test_electrons_details_call_before(): - """Test electrons for call_before details""" - test_data = output_data["test_electrons_details"]["case_call_before_1"] - response = object_test_template( - api_path=output_data["test_electrons_details"]["api_path"], - app=fastapi_app, - method_type=MethodType.GET, - path=test_data["path"], - ) - assert response.status_code == test_data["status_code"] - if "response_data" in test_data: - assert response.json() == test_data["response_data"] - - -def test_electrons_details_call_after(): - """Test electrons for call_after details""" - test_data = output_data["test_electrons_details"]["case_call_after_1"] +def test_electrons_details_hooks(): + """Test electrons for hooks details""" + test_data = output_data["test_electrons_details"]["case_hooks_1"] response = object_test_template( api_path=output_data["test_electrons_details"]["api_path"], app=fastapi_app, diff --git a/tests/covalent_ui_backend_tests/utils/assert_data/electrons.py b/tests/covalent_ui_backend_tests/utils/assert_data/electrons.py index 3930a22c5..39d04d405 100644 --- a/tests/covalent_ui_backend_tests/utils/assert_data/electrons.py +++ b/tests/covalent_ui_backend_tests/utils/assert_data/electrons.py @@ -130,33 +130,15 @@ def seed_electron_data(): "python_object": None, }, }, - "case_deps_1": { + "case_hooks_1": { "status_code": 200, "path": { "dispatch_id": VALID_DISPATCH_ID, "electron_id": VALID_NODE_ID, - "name": "deps", + "name": "hooks", }, "response_data": {"data": None, "python_object": None}, }, - "case_call_before_1": { - "status_code": 200, - "path": { - "dispatch_id": VALID_DISPATCH_ID, - "electron_id": VALID_NODE_ID, - "name": "call_before", - }, - "response_data": {"data": "", "python_object": None}, - }, - "case_call_after_1": { - "status_code": 200, - "path": { - "dispatch_id": VALID_DISPATCH_ID, - "electron_id": VALID_NODE_ID, - "name": "call_after", - }, - "response_data": {"data": "", "python_object": None}, - }, "case_error_1": { "status_code": 200, "path": { @@ -211,7 +193,7 @@ def seed_electron_data(): "electron_id": VALID_NODE_ID, "name": "results", }, - "response_message": "value is not a valid enumeration member; permitted: 'function_string', 'function', 'executor', 'result', 'value', 'stdout', 'deps', 'call_before', 'call_after', 'error', 'info', 'inputs'", + "response_message": "value is not a valid enumeration member; permitted: 'function_string', 'function', 'executor', 'result', 'value', 'stdout', 'hooks', 'error', 'info', 'inputs'", }, "case_invalid": { "status_code": 400, diff --git a/tests/covalent_ui_backend_tests/utils/data/electrons.json b/tests/covalent_ui_backend_tests/utils/data/electrons.json index 7fe8b7670..57b9f76fe 100644 --- a/tests/covalent_ui_backend_tests/utils/data/electrons.json +++ b/tests/covalent_ui_backend_tests/utils/data/electrons.json @@ -1,10 +1,8 @@ [ { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.483405", "created_at": "2022-09-23 10:01:11.062647", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -32,11 +30,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.520838", "created_at": "2022-09-23 10:01:11.075465", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -64,11 +60,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.194419", "created_at": "2022-09-23 10:01:11.085971", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -96,11 +90,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.588248", "created_at": "2022-09-23 10:01:11.098325", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -128,11 +120,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.640049", "created_at": "2022-09-23 10:01:11.109305", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -160,11 +150,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.226986", "created_at": "2022-09-23 10:01:11.121100", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data": {}, @@ -206,9 +194,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:33.721544", @@ -238,9 +224,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:33.737909", @@ -270,9 +254,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:33.754840", @@ -302,9 +284,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:33.769064", @@ -334,9 +314,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.121824", @@ -366,9 +344,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.141566", @@ -398,9 +374,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.157807", @@ -430,9 +404,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.174837", @@ -462,9 +434,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.193334", @@ -494,9 +464,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.210810", @@ -526,9 +494,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.225974", @@ -558,9 +524,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.243921", @@ -590,9 +554,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.262261", @@ -622,9 +584,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.279546", @@ -654,9 +614,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.300588", @@ -686,9 +644,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.319777", @@ -718,9 +674,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.337839", @@ -750,9 +704,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.357823", @@ -782,9 +734,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.374314", @@ -814,9 +764,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.390256", @@ -846,9 +794,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.406105", @@ -878,9 +824,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.424469", @@ -910,9 +854,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.442225", @@ -942,9 +884,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:34.457399", @@ -974,9 +914,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.302082", @@ -1006,9 +944,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.320779", @@ -1038,9 +974,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.338612", @@ -1070,9 +1004,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.355690", @@ -1102,9 +1034,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.373101", @@ -1134,9 +1064,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.389371", @@ -1166,9 +1094,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.407041", @@ -1198,9 +1124,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.426321", @@ -1230,9 +1154,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.442352", @@ -1262,9 +1184,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.457450", @@ -1294,9 +1214,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.474123", @@ -1326,9 +1244,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.489691", @@ -1358,9 +1274,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.508357", @@ -1390,9 +1304,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.526014", @@ -1422,9 +1334,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.543830", @@ -1454,9 +1364,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.558599", @@ -1486,9 +1394,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.574942", @@ -1518,9 +1424,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.592225", @@ -1550,9 +1454,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.605931", @@ -1582,9 +1484,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.624526", @@ -1614,9 +1514,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.642423", @@ -1646,9 +1544,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.659032", @@ -1678,9 +1574,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.675982", @@ -1710,9 +1604,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.691008", @@ -1742,9 +1634,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.706843", @@ -1774,9 +1664,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.725044", @@ -1806,9 +1694,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.742116", @@ -1838,9 +1724,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.758671", @@ -1870,9 +1754,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.778892", @@ -1902,9 +1784,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.795931", @@ -1934,9 +1814,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.808616", @@ -1966,9 +1844,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.826128", @@ -1998,9 +1874,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.842655", @@ -2030,9 +1904,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.857408", @@ -2062,9 +1934,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.875349", @@ -2094,9 +1964,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.892565", @@ -2126,9 +1994,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.909599", @@ -2158,9 +2024,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.925895", @@ -2190,9 +2054,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.941241", @@ -2222,9 +2084,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.958115", @@ -2254,9 +2114,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.972882", @@ -2286,9 +2144,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:36.988747", @@ -2318,9 +2174,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.004250", @@ -2350,9 +2204,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.024965", @@ -2382,9 +2234,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.042155", @@ -2414,9 +2264,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.057877", @@ -2446,9 +2294,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.074699", @@ -2478,9 +2324,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.090017", @@ -2510,9 +2354,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.108724", @@ -2542,9 +2384,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.127189", @@ -2574,9 +2414,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.148132", @@ -2606,9 +2444,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.163845", @@ -2638,9 +2474,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.182859", @@ -2670,9 +2504,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.197103", @@ -2702,9 +2534,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.212968", @@ -2734,9 +2564,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.233015", @@ -2766,9 +2594,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.250635", @@ -2798,9 +2624,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.272024", @@ -2830,9 +2654,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.285661", @@ -2862,9 +2684,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.295628", @@ -2894,9 +2714,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.308194", @@ -2926,9 +2744,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.330291", @@ -2958,9 +2774,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.351111", @@ -2990,9 +2804,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.374099", @@ -3022,9 +2834,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.392103", @@ -3054,9 +2864,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.409575", @@ -3086,9 +2894,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.429387", @@ -3118,9 +2924,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.448684", @@ -3150,9 +2954,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.470073", @@ -3182,9 +2984,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.487636", @@ -3214,9 +3014,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.504586", @@ -3246,9 +3044,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.526051", @@ -3278,9 +3074,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.546764", @@ -3310,9 +3104,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.567169", @@ -3342,9 +3134,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.585364", @@ -3374,9 +3164,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.599294", @@ -3406,9 +3194,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.619618", @@ -3438,9 +3224,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.638808", @@ -3470,9 +3254,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.654483", @@ -3502,9 +3284,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.671724", @@ -3534,9 +3314,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.687139", @@ -3566,9 +3344,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.704070", @@ -3598,9 +3374,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.725651", @@ -3630,9 +3404,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.745048", @@ -3662,9 +3434,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.760175", @@ -3694,9 +3464,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.776605", @@ -3726,9 +3494,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.792093", @@ -3758,9 +3524,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.808215", @@ -3790,9 +3554,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.826520", @@ -3822,9 +3584,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.842543", @@ -3854,9 +3614,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.859025", @@ -3886,9 +3644,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.875312", @@ -3918,9 +3674,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.890211", @@ -3950,9 +3704,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.906062", @@ -3982,9 +3734,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.925923", @@ -4014,9 +3764,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.941795", @@ -4046,9 +3794,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.957926", @@ -4078,9 +3824,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.973121", @@ -4110,9 +3854,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:37.986933", @@ -4142,9 +3884,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.001766", @@ -4174,9 +3914,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.020035", @@ -4206,9 +3944,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.035546", @@ -4238,9 +3974,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.046972", @@ -4270,9 +4004,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.059033", @@ -4302,9 +4034,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.073850", @@ -4334,9 +4064,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.086138", @@ -4366,9 +4094,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.098049", @@ -4398,9 +4124,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.112014", @@ -4430,9 +4154,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.129412", @@ -4462,9 +4184,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.146475", @@ -4494,9 +4214,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.162055", @@ -4526,9 +4244,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.178884", @@ -4558,9 +4274,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.195346", @@ -4590,9 +4304,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.214888", @@ -4622,9 +4334,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.237282", @@ -4654,9 +4364,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.253156", @@ -4686,9 +4394,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.271635", @@ -4718,9 +4424,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.287718", @@ -4750,9 +4454,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.302767", @@ -4782,9 +4484,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "is_active": 1, "created_at": "2022-10-27 10:08:38.320742", @@ -4800,11 +4500,9 @@ "error_filename": "error.log" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2023-08-10 10:08:55.902257", "created_at": "2023-08-10 10:08:55.403522", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "error_filename": "error.log", "executor": "dask", "executor_data": {}, @@ -4832,11 +4530,9 @@ "value_filename": "value.pkl" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2023-08-10 10:08:55.822223", "created_at": "2023-08-10 10:08:55.396776", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "error_filename": "error.log", "executor": "dask", "executor_data": {}, diff --git a/tests/covalent_ui_backend_tests/utils/data/electrons_old.json b/tests/covalent_ui_backend_tests/utils/data/electrons_old.json index afba66245..34437f38e 100644 --- a/tests/covalent_ui_backend_tests/utils/data/electrons_old.json +++ b/tests/covalent_ui_backend_tests/utils/data/electrons_old.json @@ -1,10 +1,8 @@ [ { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.483405", "created_at": "2022-09-23 10:01:11.062647", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -29,11 +27,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.520838", "created_at": "2022-09-23 10:01:11.075465", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -58,11 +54,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.194419", "created_at": "2022-09-23 10:01:11.085971", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -87,11 +81,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.588248", "created_at": "2022-09-23 10:01:11.098325", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -116,11 +108,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.640049", "created_at": "2022-09-23 10:01:11.109305", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -145,11 +135,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.226986", "created_at": "2022-09-23 10:01:11.121100", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "executor": "dask", "executor_data_filename": "executor_data.pkl", "function_filename": "function.pkl", @@ -187,9 +175,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -216,9 +202,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -245,9 +229,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -274,9 +256,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -303,9 +283,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -332,9 +310,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -361,9 +337,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -390,9 +364,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -419,9 +391,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -448,9 +418,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -477,9 +445,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -506,9 +472,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -535,9 +499,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -564,9 +526,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -593,9 +553,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -622,9 +580,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -651,9 +607,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -680,9 +634,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -709,9 +661,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -738,9 +688,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -767,9 +715,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -796,9 +742,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -825,9 +769,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -854,9 +796,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -883,9 +823,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -912,9 +850,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -941,9 +877,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -970,9 +904,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -999,9 +931,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1028,9 +958,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1057,9 +985,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1086,9 +1012,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1115,9 +1039,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1144,9 +1066,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1173,9 +1093,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1202,9 +1120,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1231,9 +1147,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1260,9 +1174,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1289,9 +1201,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1318,9 +1228,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1347,9 +1255,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1376,9 +1282,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1405,9 +1309,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1434,9 +1336,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1463,9 +1363,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1492,9 +1390,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1521,9 +1417,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1550,9 +1444,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1579,9 +1471,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1608,9 +1498,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1637,9 +1525,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1666,9 +1552,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1695,9 +1579,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1724,9 +1606,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1753,9 +1633,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1782,9 +1660,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1811,9 +1687,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1840,9 +1714,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1869,9 +1741,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1898,9 +1768,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1927,9 +1795,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1956,9 +1822,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -1985,9 +1849,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2014,9 +1876,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2043,9 +1903,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2072,9 +1930,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2101,9 +1957,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2130,9 +1984,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2159,9 +2011,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2188,9 +2038,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2217,9 +2065,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2246,9 +2092,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2275,9 +2119,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2304,9 +2146,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2333,9 +2173,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2362,9 +2200,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2391,9 +2227,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2420,9 +2254,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2449,9 +2281,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2478,9 +2308,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2507,9 +2335,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2536,9 +2362,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2565,9 +2389,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2594,9 +2416,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2623,9 +2443,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2652,9 +2470,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2681,9 +2497,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2710,9 +2524,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2739,9 +2551,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2768,9 +2578,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2797,9 +2605,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2826,9 +2632,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2855,9 +2659,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2884,9 +2686,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2913,9 +2713,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2942,9 +2740,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -2971,9 +2767,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3000,9 +2794,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3029,9 +2821,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3058,9 +2848,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3087,9 +2875,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3116,9 +2902,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3145,9 +2929,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3174,9 +2956,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3203,9 +2983,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3232,9 +3010,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3261,9 +3037,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3290,9 +3064,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3319,9 +3091,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3348,9 +3118,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3377,9 +3145,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3406,9 +3172,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3435,9 +3199,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3464,9 +3226,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3493,9 +3253,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3522,9 +3280,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3551,9 +3307,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3580,9 +3334,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3609,9 +3361,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3638,9 +3388,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3667,9 +3415,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3696,9 +3442,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3725,9 +3469,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3754,9 +3496,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3783,9 +3523,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3812,9 +3550,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3841,9 +3577,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3870,9 +3604,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3899,9 +3631,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3928,9 +3658,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3957,9 +3685,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -3986,9 +3712,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4015,9 +3739,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4044,9 +3766,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4073,9 +3793,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4102,9 +3820,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4131,9 +3847,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4160,9 +3874,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4189,9 +3901,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4218,9 +3928,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4247,9 +3955,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4276,9 +3982,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4305,9 +4009,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4334,9 +4036,7 @@ "results_filename": "results.pkl", "value_filename": "value.pkl", "stdout_filename": "stdout.log", - "deps_filename": "deps.pkl", - "call_before_filename": "call_before.pkl", - "call_after_filename": "call_after.pkl", + "hooks_filename": "hooks.pkl", "stderr_filename": "stderr.log", "info_filename": "info.log", "is_active": 1, @@ -4350,11 +4050,9 @@ "qelectron_data_exists": false }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2023-08-10 10:08:55.822223", "created_at": "2023-08-10 10:08:55.396776", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "error_filename": "error.log", "executor": "dask", "executor_data_filename": "executor_data.pkl", @@ -4379,11 +4077,9 @@ "value_filename": "value.pkl" }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2023-08-10 10:08:55.902257", "created_at": "2023-08-10 10:08:55.403522", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "error_filename": "error.log", "executor": "dask", "executor_data_filename": "executor_data.pkl", diff --git a/tests/covalent_ui_backend_tests/utils/data/lattices.json b/tests/covalent_ui_backend_tests/utils/data/lattices.json index bbe1fa9d3..a0d953242 100644 --- a/tests/covalent_ui_backend_tests/utils/data/lattices.json +++ b/tests/covalent_ui_backend_tests/utils/data/lattices.json @@ -1,12 +1,10 @@ [ { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-09-23 10:01:11.717064", "completed_electron_num": 6, "cova_imports_filename": "cova_imports.json", "created_at": "2022-09-23 10:01:11.044857", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "dispatch_id": "78525234-72ec-42dc-94a0-f4751707f9cd", "docstring_filename": "function_docstring.txt", "electron_id": null, @@ -37,13 +35,11 @@ "workflow_executor_data": {} }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-10-27 10:08:43.991108", "completed_electron_num": 4, "cova_imports_filename": "cova_imports.json", "created_at": "2022-10-27 10:08:33.702548", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "dispatch_id": "a95d84ad-c441-446d-83ae-46380dcdf38e", "docstring_filename": "function_docstring.txt", "electron_id": null, @@ -74,13 +70,11 @@ "workflow_executor_data": {} }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-10-27 10:08:35.997225", "completed_electron_num": 20, "cova_imports_filename": "cova_imports.json", "created_at": "2022-10-27 10:08:34.103977", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "dispatch_id": "89be0bcf-95dd-40a6-947e-6af6c56f147d", "docstring_filename": "function_docstring.txt", "electron_id": 9, @@ -111,13 +105,11 @@ "workflow_executor_data": {} }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2022-10-27 10:08:43.877056", "completed_electron_num": 120, "cova_imports_filename": "cova_imports.json", "created_at": "2022-10-27 10:08:36.287047", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "dispatch_id": "69dec597-79d9-4c99-96de-8d5f06f3d4dd", "docstring_filename": "function_docstring.txt", "electron_id": 10, @@ -148,13 +140,11 @@ "workflow_executor_data": {} }, { - "call_after_filename": "call_after.pkl", - "call_before_filename": "call_before.pkl", "completed_at": "2023-08-10 10:08:55.902257", "completed_electron_num": 2, "cova_imports_filename": "cova_imports.json", "created_at": "2023-08-10 10:08:55.387554", - "deps_filename": "deps.pkl", + "hooks_filename": "hooks.pkl", "dispatch_id": "e8fd09c9-1406-4686-9e77-c8d4d64a76ee", "docstring_filename": "function_docstring.txt", "electron_id": null, diff --git a/tests/covalent_ui_backend_tests/utils/seed_script.py b/tests/covalent_ui_backend_tests/utils/seed_script.py index da24b03e4..66a4bd158 100644 --- a/tests/covalent_ui_backend_tests/utils/seed_script.py +++ b/tests/covalent_ui_backend_tests/utils/seed_script.py @@ -49,10 +49,7 @@ def seed(engine): electron_num=item["electron_num"], completed_electron_num=item["completed_electron_num"], docstring_filename=item["docstring_filename"], - deps_filename=item["deps_filename"], - call_before_filename=item["call_before_filename"], - call_after_filename=item["call_after_filename"], - cova_imports_filename=item["cova_imports_filename"], + hooks_filename=item["hooks_filename"], lattice_imports_filename=item["lattice_imports_filename"], storage_type=item["storage_type"], storage_path=os.path.dirname(__file__) @@ -106,9 +103,7 @@ def seed(engine): results_filename=item["results_filename"], value_filename=item["value_filename"], stdout_filename=item["stdout_filename"], - deps_filename=item["deps_filename"], - call_before_filename=item["call_before_filename"], - call_after_filename=item["call_after_filename"], + hooks_filename=item["hooks_filename"], stderr_filename=item["stderr_filename"], error_filename=item["error_filename"], is_active=item["is_active"], diff --git a/tests/functional_tests/workflow_stack_test.py b/tests/functional_tests/workflow_stack_test.py index 5d76993b3..47e1578a7 100644 --- a/tests/functional_tests/workflow_stack_test.py +++ b/tests/functional_tests/workflow_stack_test.py @@ -25,7 +25,6 @@ import covalent._dispatcher_plugins.local as local import covalent._results_manager.results_manager as rm from covalent._results_manager.result import Result -from covalent.executor import LocalExecutor def construct_temp_cache_dir(): @@ -140,7 +139,7 @@ def task_1(): def task_2(): pass - @ct.lattice(executor=LocalExecutor(), deps_bash=l_bash_dep) + @ct.lattice(deps_bash=l_bash_dep) def workflow(): task_1() task_2() @@ -149,8 +148,8 @@ def workflow(): res = ct.get_result(dispatch_id, wait=True) tg = res.lattice.transport_graph - dep_0 = ct.DepsBash().from_dict(tg.get_node_value(0, "metadata")["deps"]["bash"]) - dep_1 = ct.DepsBash().from_dict(tg.get_node_value(1, "metadata")["deps"]["bash"]) + dep_0 = ct.DepsBash().from_dict(tg.get_node_value(0, "metadata")["hooks"]["deps"]["bash"]) + dep_1 = ct.DepsBash().from_dict(tg.get_node_value(1, "metadata")["hooks"]["deps"]["bash"]) assert dep_0.commands == ["ls"] assert dep_1.commands == ["ls -l"] From d94bdaf6ef2b126dde05d60e26eab23bd913e28a Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Mon, 11 Dec 2023 16:05:52 -0500 Subject: [PATCH 5/6] Minor fixes to pydantic models (#1884) * Minor fixes encountered recently * changelog updated * removed import error suppression from covalent init for qelectrons --- CHANGELOG.md | 2 ++ covalent/_shared_files/schemas/electron.py | 4 ++-- covalent/_shared_files/schemas/lattice.py | 4 ++-- covalent/_workflow/transport.py | 2 +- requirements-client.txt | 1 + 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec165d88b..0dd0d6b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 important, whether the size is reported to be zero or positive does have consequences. - Pack deps, call_before, and call_after assets into one file. +- `qelectron_db`, `qelectron_data_exists`, `python_version`, and `covalent_version` + are now optional in the pydantic model definitions. ### Fixed diff --git a/covalent/_shared_files/schemas/electron.py b/covalent/_shared_files/schemas/electron.py index 4a6541383..b245cc93d 100644 --- a/covalent/_shared_files/schemas/electron.py +++ b/covalent/_shared_files/schemas/electron.py @@ -86,7 +86,7 @@ class ElectronAssets(BaseModel): error: Optional[AssetSchema] = None stdout: Optional[AssetSchema] = None stderr: Optional[AssetSchema] = None - qelectron_db: AssetSchema + qelectron_db: Optional[AssetSchema] = None # user dependent assets hooks: AssetSchema @@ -97,7 +97,7 @@ class ElectronMetadata(BaseModel): name: str executor: str executor_data: dict - qelectron_data_exists: bool + qelectron_data_exists: Optional[bool] = None sub_dispatch_id: Optional[str] = None status: StatusEnum start_time: Optional[datetime] = None diff --git a/covalent/_shared_files/schemas/lattice.py b/covalent/_shared_files/schemas/lattice.py index 0fdf35f16..6a3e2bbf9 100644 --- a/covalent/_shared_files/schemas/lattice.py +++ b/covalent/_shared_files/schemas/lattice.py @@ -98,8 +98,8 @@ class LatticeMetadata(BaseModel): executor_data: dict workflow_executor: str workflow_executor_data: dict - python_version: str - covalent_version: str + python_version: Optional[str] = None + covalent_version: Optional[str] = None class LatticeSchema(BaseModel): diff --git a/covalent/_workflow/transport.py b/covalent/_workflow/transport.py index 2ab1db739..77ed5a14d 100644 --- a/covalent/_workflow/transport.py +++ b/covalent/_workflow/transport.py @@ -212,7 +212,7 @@ def get_node_value(self, node_key: int, value_key: str) -> Any: """ return self._graph.nodes[node_key][value_key] - def set_node_value(self, node_key: int, value_key: int, value: Any) -> None: + def set_node_value(self, node_key: str, value_key: int, value: Any) -> None: """ Set a certain value of a node. This allows for saving custom data in the graph nodes. diff --git a/requirements-client.txt b/requirements-client.txt index 9f7e806aa..ede6a20e3 100644 --- a/requirements-client.txt +++ b/requirements-client.txt @@ -5,6 +5,7 @@ dask[distributed]>=2022.6.0 filelock>=3.12.2 furl>=2.1.3 networkx>=2.8.6 +pydantic>=2.1.1 requests>=2.24.0 simplejson>=3.17.6 toml>=0.10.2 From 6e331d4b1fef702f5a29c9f7d6fe85c727c684a9 Mon Sep 17 00:00:00 2001 From: Sankalp Sanand Date: Mon, 11 Dec 2023 16:49:12 -0500 Subject: [PATCH 6/6] Adding Module Dependencies feature (#1876) * ran precommit * first draft of modeule deps * updated changelog * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * added support for directly adding modules * minor edit * debugging and trying * now working as expected except for postprocess * some significant size improvements * fixed postprocess issue * removed unnecesary moduledeps additions * fixed one electron metadata issue * removed some unneeded changes * removed TG size printing * added transport.py tests * added tests for electron file changes * fixed tests * fixing more tests --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 5 + covalent/__init__.py | 1 + covalent/_workflow/__init__.py | 1 + covalent/_workflow/depscall.py | 15 +-- covalent/_workflow/depsmodule.py | 56 ++++++++++++ covalent/_workflow/electron.py | 31 ++++++- covalent/_workflow/lattice.py | 16 +++- covalent/_workflow/postprocessing.py | 1 + covalent/_workflow/transport.py | 91 ++++++++++++++++++- .../covalent_tests/workflow/electron_test.py | 27 ++++++ .../covalent_tests/workflow/transport_test.py | 58 +++++++++++- 11 files changed, 284 insertions(+), 18 deletions(-) create mode 100644 covalent/_workflow/depsmodule.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd0d6b66..594c1772e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Added + +- Added feature to use custom python files as modules to be used in the electron function + ### Changed - SDK no longer uploads empty assets when submitting a dispatch. @@ -18,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 important, whether the size is reported to be zero or positive does have consequences. - Pack deps, call_before, and call_after assets into one file. +- Changed handling of tuples and sets when building the transport graph - they will be converted to electron lists as well for now - `qelectron_db`, `qelectron_data_exists`, `python_version`, and `covalent_version` are now optional in the pydantic model definitions. diff --git a/covalent/__init__.py b/covalent/__init__.py index 524952776..6d88af00d 100644 --- a/covalent/__init__.py +++ b/covalent/__init__.py @@ -40,6 +40,7 @@ from ._workflow import ( # nopycln: import DepsBash, DepsCall, + DepsModule, DepsPip, Lepton, TransportableObject, diff --git a/covalent/_workflow/__init__.py b/covalent/_workflow/__init__.py index 98da8e676..64ce547c8 100644 --- a/covalent/_workflow/__init__.py +++ b/covalent/_workflow/__init__.py @@ -18,6 +18,7 @@ from .depsbash import DepsBash from .depscall import DepsCall +from .depsmodule import DepsModule from .depspip import DepsPip from .electron import electron from .lattice import lattice diff --git a/covalent/_workflow/depscall.py b/covalent/_workflow/depscall.py index 9e4931c08..48d009f31 100644 --- a/covalent/_workflow/depscall.py +++ b/covalent/_workflow/depscall.py @@ -51,14 +51,18 @@ class DepsCall(Deps): def __init__( self, func=None, - args=[], - kwargs={}, + args=None, + kwargs=None, *, retval_keyword="", override_reserved_retval_keys=False, ): + if args is None: + args = [] + if kwargs is None: + kwargs = {} if not override_reserved_retval_keys and retval_keyword in [RESERVED_RETVAL_KEY__FILES]: - raise Exception( + raise RuntimeError( f"The retval_keyword for the specified DepsCall uses the reserved value '{retval_keyword}' please re-name to use another return value keyword." ) @@ -70,10 +74,7 @@ def to_dict(self) -> dict: """Return a JSON-serializable dictionary representation of self""" attributes = self.__dict__.copy() for k, v in attributes.items(): - if isinstance(v, TransportableObject): - attributes[k] = v.to_dict() - else: - attributes[k] = v + attributes[k] = v.to_dict() if isinstance(v, TransportableObject) else v return {"type": "DepsCall", "short_name": self.short_name(), "attributes": attributes} def from_dict(self, object_dict) -> "DepsCall": diff --git a/covalent/_workflow/depsmodule.py b/covalent/_workflow/depsmodule.py new file mode 100644 index 000000000..976df8e63 --- /dev/null +++ b/covalent/_workflow/depsmodule.py @@ -0,0 +1,56 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from types import ModuleType +from typing import Union + +from .depscall import DepsCall +from .transportable_object import TransportableObject + + +class DepsModule(DepsCall): + """ + Python modules to be imported in an electron's execution environment. + + This is only used as a vehicle to send the module by reference to the + to the right place of serialization where it will instead be pickled by value. + + Deps class to encapsulate python modules to be + imported in the same execution environment as the electron. + + Note: This subclasses the DepsCall class due to its pre-existing + infrastructure integrations, and not because of its logical functionality. + + Attributes: + module_name: A string containing the name of the module to be imported. + """ + + def __init__(self, module: Union[str, ModuleType]): + if isinstance(module, str): + # Import the module on the client side + module = importlib.import_module(module) + + # Temporarily pickling the module by reference + # so that it can be pickled by value when serializing + # the transport graph. + self.pickled_module = TransportableObject(module) + + super().__init__() + + def short_name(self): + """Returns the short name of this class.""" + return "depsmodule" diff --git a/covalent/_workflow/electron.py b/covalent/_workflow/electron.py index b772a8954..12f18cbf5 100644 --- a/covalent/_workflow/electron.py +++ b/covalent/_workflow/electron.py @@ -24,6 +24,7 @@ from copy import deepcopy from dataclasses import asdict from functools import wraps +from types import ModuleType from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Union from covalent._dispatcher_plugins.local import LocalDispatcher @@ -49,6 +50,7 @@ ) from .depsbash import DepsBash from .depscall import RESERVED_RETVAL_KEY__FILES, DepsCall +from .depsmodule import DepsModule from .depspip import DepsPip from .lattice import Lattice from .transport import TransportableObject, encode_metadata @@ -207,6 +209,7 @@ def func_for_op(arg_1: Union[Any, "Electron"], arg_2: Union[Any, "Electron"]) -> metadata = encode_metadata(DEFAULT_METADATA_VALUES.copy()) executor = metadata["workflow_executor"] executor_data = metadata["workflow_executor_data"] + op_electron = Electron(func_for_op, metadata=metadata) if active_lattice := active_lattice_manager.get_active_lattice(): @@ -545,8 +548,8 @@ def connect_node_with_others( arg_index=arg_index, ) - elif isinstance(param_value, list): - + elif isinstance(param_value, (list, tuple, set)): + # Tuples and sets will also be converted to lists def _auto_list_node(*args, **kwargs): return list(args) @@ -589,6 +592,7 @@ def _auto_dict_node(*args, **kwargs): else: encoded_param_value = TransportableObject.make_transportable(param_value) + parameter_node = transport_graph.add_node( name=parameter_prefix + str(param_value), function=None, @@ -698,6 +702,7 @@ def electron( files: List[FileTransfer] = [], deps_bash: Union[DepsBash, List, str] = None, deps_pip: Union[DepsPip, list] = None, + deps_module: Union[DepsModule, List[DepsModule], str, List[str]] = None, call_before: Union[List[DepsCall], DepsCall] = None, call_after: Union[List[DepsCall], DepsCall] = None, ) -> Callable: # sourcery skip: assign-if-exp @@ -713,6 +718,7 @@ def electron( executor is used by default. deps_bash: An optional DepsBash object specifying a list of shell commands to run before `_func` deps_pip: An optional DepsPip object specifying a list of PyPI packages to install before running `_func` + deps_module: An optional DepsModule (or similar) object specifying which user modules to load before running `_func` call_before: An optional list of DepsCall objects specifying python functions to invoke before the electron call_after: An optional list of DepsCall objects specifying python functions to invoke after the electron files: An optional list of FileTransfer objects which copy files to/from remote or local filesystems. @@ -757,6 +763,25 @@ def electron( else: internal_call_before_deps.append(DepsCall(_file_transfer_call_dep_)) + if deps_module: + if isinstance(deps_module, list): + # Convert to DepsModule objects + converted_deps = [] + for dep in deps_module: + if type(dep) in [str, ModuleType]: + converted_deps.append(DepsModule(dep)) + else: + converted_deps.append(dep) + deps_module = converted_deps + + elif type(deps_module) in [str, ModuleType]: + deps_module = [DepsModule(deps_module)] + + elif isinstance(deps_module, DepsModule): + deps_module = [deps_module] + + internal_call_before_deps.extend(deps_module) + if isinstance(deps_pip, DepsPip): deps["pip"] = deps_pip if isinstance(deps_pip, list): @@ -887,7 +912,7 @@ def _build_sublattice_graph(sub: Lattice, json_parent_metadata: str, *args, **kw ) LocalDispatcher.upload_assets(recv_manifest) - return recv_manifest.json() + return recv_manifest.model_dump_json() except Exception as ex: # Fall back to legacy sublattice handling diff --git a/covalent/_workflow/lattice.py b/covalent/_workflow/lattice.py index 8f81fb706..84f74f6b1 100644 --- a/covalent/_workflow/lattice.py +++ b/covalent/_workflow/lattice.py @@ -35,7 +35,12 @@ from .depscall import DepsCall from .depspip import DepsPip from .postprocessing import Postprocessor -from .transport import TransportableObject, _TransportGraph, encode_metadata +from .transport import ( + TransportableObject, + _TransportGraph, + add_module_deps_to_lattice_metadata, + encode_metadata, +) if TYPE_CHECKING: from .._results_manager.result import Result @@ -234,10 +239,11 @@ def build_graph(self, *args, **kwargs) -> None: pp = Postprocessor(lattice=self) - if get_config("sdk.exhaustive_postprocess") == "true": - pp.add_exhaustive_postprocess_node(self._bound_electrons.copy()) - else: - pp.add_reconstruct_postprocess_node(retval, self._bound_electrons.copy()) + with add_module_deps_to_lattice_metadata(pp, self._bound_electrons): + if get_config("sdk.exhaustive_postprocess") == "true": + pp.add_exhaustive_postprocess_node(self._bound_electrons.copy()) + else: + pp.add_reconstruct_postprocess_node(retval, self._bound_electrons.copy()) self._bound_electrons = {} # Reset bound electrons diff --git a/covalent/_workflow/postprocessing.py b/covalent/_workflow/postprocessing.py index 1072cd9d5..c00a87155 100644 --- a/covalent/_workflow/postprocessing.py +++ b/covalent/_workflow/postprocessing.py @@ -149,6 +149,7 @@ def _get_electron_metadata(self) -> Dict: pp_metadata["executor"] = pp_metadata.pop("workflow_executor") pp_metadata["executor_data"] = pp_metadata.pop("workflow_executor_data") + return pp_metadata def add_exhaustive_postprocess_node(self, bound_electrons: Dict) -> None: diff --git a/covalent/_workflow/transport.py b/covalent/_workflow/transport.py index 77ed5a14d..d3007e0e2 100644 --- a/covalent/_workflow/transport.py +++ b/covalent/_workflow/transport.py @@ -18,6 +18,7 @@ import datetime import json +from contextlib import contextmanager from copy import deepcopy from typing import Any, Callable, Dict @@ -94,6 +95,88 @@ def encode_metadata(metadata: dict) -> dict: return encoded_metadata +@contextmanager +def pickle_modules_by_value(metadata): + """ + Pickle modules in a context manager by value. + + Args: + call_before: The call before metadata. + + Returns: + None + """ + + call_before = metadata.get("hooks", {}).get("call_before") + + if not call_before: + yield metadata + return + + new_metadata = deepcopy(metadata) + call_before = new_metadata.get("hooks", {}).get("call_before") + + list_of_modules = [] + + for i in range(len(call_before)): + # Extract the pickled module if the call before is a DepsModule + if call_before[i]["short_name"] == "depsmodule": + pickled_module = call_before[i]["attributes"]["pickled_module"] + list_of_modules.append( + TransportableObject.from_dict(pickled_module).get_deserialized() + ) + + # Delete the DepsModule from new_metadata of the electron + new_metadata["hooks"]["call_before"][i] = None + + # Remove the None values from the call_before list of this electron + new_metadata["hooks"]["call_before"] = list(filter(None, new_metadata["hooks"]["call_before"])) + + for module in list_of_modules: + cloudpickle.register_pickle_by_value(module) + + yield new_metadata + + for module in list_of_modules: + try: + cloudpickle.unregister_pickle_by_value(module) + except ValueError: + continue + + +@contextmanager +def add_module_deps_to_lattice_metadata(pp, bound_electrons: Dict[int, Any]): + old_lattice_metadata = deepcopy(pp.lattice.metadata) + + # Add the module dependencies to the lattice metadata + for electron in bound_electrons.values(): + call_before = electron.metadata.get("hooks", {}).get("call_before") + if call_before: + for i in range(len(call_before)): + if call_before[i]["short_name"] == "depsmodule": + if "hooks" not in pp.lattice.metadata: + pp.lattice.metadata["hooks"] = {} + + if "call_before" not in pp.lattice.metadata["hooks"]: + pp.lattice.metadata["hooks"]["call_before"] = [] + + # Temporarily add the module metadat to the lattice metadata for postprocessing + pp.lattice.metadata["hooks"]["call_before"].append(call_before[i]) + + # Delete the DepsModule from the electron metadata + electron.metadata["hooks"]["call_before"][i] = None + + # Remove the None values from the call_before list of this electron + electron.metadata["hooks"]["call_before"] = list( + filter(None, electron.metadata["hooks"]["call_before"]) + ) + + yield + + # Restore the old lattice metadata + pp.lattice.metadata = old_lattice_metadata + + class _TransportGraph: """ A TransportGraph is the most essential part of the whole workflow. This contains @@ -131,6 +214,7 @@ def add_node( ) -> int: """ Adds a node to the graph. + Also serializes the received function. Args: name: The name of the node. @@ -148,14 +232,17 @@ def add_node( if task_group_id is None: task_group_id = node_id + with pickle_modules_by_value(metadata) as new_metadata: + serialized_function = TransportableObject(function) + # Default to gid=node_id self._graph.add_node( node_id, task_group_id=task_group_id, name=name, - function=TransportableObject(function), - metadata=metadata, + function=serialized_function, + metadata=new_metadata, # Save the new metadata without the DepsModules in it **attr, ) diff --git a/tests/covalent_tests/workflow/electron_test.py b/tests/covalent_tests/workflow/electron_test.py index 25bba748e..a3db76bb7 100644 --- a/tests/covalent_tests/workflow/electron_test.py +++ b/tests/covalent_tests/workflow/electron_test.py @@ -19,6 +19,8 @@ import json from unittest.mock import ANY, MagicMock +import flake8 +import isort import pytest import covalent as ct @@ -683,3 +685,28 @@ def workflow(x): workflow.build_graph(2) mock_electron_get_op_function.assert_called_with(ANY, 2, "**") + + +@pytest.mark.parametrize( + "module_inputs", + [ + "isort", + isort, + ct.DepsModule("isort"), + ["isort", "flake8"], + [isort, flake8], + [ct.DepsModule("isort"), ct.DepsModule("flake8")], + ], +) +def test_deps_modules_are_added(module_inputs): + """Test that deps modules are added to the lattice metadata.""" + + @ct.electron(deps_module=module_inputs) + def task(x): + return x + + # Making sure all kinds of inputs are converted to DepsModule + # and then to its transportable form of a dictionary + for cb in task.electron_object.metadata["hooks"]["call_before"]: + assert cb["short_name"] == "depsmodule" + assert cb["attributes"]["pickled_module"]["type"] == "TransportableObject" diff --git a/tests/covalent_tests/workflow/transport_test.py b/tests/covalent_tests/workflow/transport_test.py index 4c9ce6855..40de076c1 100644 --- a/tests/covalent_tests/workflow/transport_test.py +++ b/tests/covalent_tests/workflow/transport_test.py @@ -25,7 +25,13 @@ import covalent as ct from covalent._shared_files.defaults import parameter_prefix -from covalent._workflow.transport import TransportableObject, _TransportGraph, encode_metadata +from covalent._workflow.transport import ( + TransportableObject, + _TransportGraph, + add_module_deps_to_lattice_metadata, + encode_metadata, + pickle_modules_by_value, +) from covalent.executor import LocalExecutor from covalent.triggers import BaseTrigger @@ -469,3 +475,53 @@ def test_reset_node(workflow_transport_graph, mocker): for mock_call in expected_mock_calls: assert mock_call in actual_mock_calls + + +@pytest.mark.parametrize( + ["call_before", "metadata", "expected_metadata"], + [ + ([], {}, {}), + ( + [ct.DepsModule("isort").to_dict()], + {"hooks": {"call_before": [ct.DepsModule("isort").to_dict()]}}, + {"hooks": {"call_before": []}}, + ), + ], +) +def test_pickle_modules_by_value(mocker, call_before, metadata, expected_metadata): + """ + Test that the modules mentioned in + DepsModules are pickled by value. + """ + import isort + + mock_cloudpickle = mocker.patch("covalent._workflow.transport.cloudpickle") + + with pickle_modules_by_value(metadata) as received_metadata: + assert received_metadata == expected_metadata + + if call_before: + mock_cloudpickle.register_pickle_by_value.assert_called_once_with(isort) + mock_cloudpickle.unregister_pickle_by_value.assert_called_once_with(isort) + + +def test_add_module_deps_to_lattice_metadata(mocker): + """ + Test that the modules mentioned in + DepsModules are added to the lattice metadata temporarily. + """ + + pp = mocker.Mock() + pp.lattice.metadata = {"hooks": {"call_before": []}} + + mock_call_before = [ct.DepsModule("isort").to_dict()] + + mock_electron = mocker.Mock() + mock_electron.metadata = {"hooks": {"call_before": mock_call_before.copy()}} + + mock_bound_electrons = {0: mock_electron} + + with add_module_deps_to_lattice_metadata(pp, mock_bound_electrons): + assert pp.lattice.metadata["hooks"]["call_before"] == mock_call_before + + assert pp.lattice.metadata["hooks"]["call_before"] == []