diff --git a/.github/workflows/boilerplate.yml b/.github/workflows/boilerplate.yml index 13a2f111a..862b33f87 100644 --- a/.github/workflows/boilerplate.yml +++ b/.github/workflows/boilerplate.yml @@ -85,9 +85,26 @@ jobs: # See the License for the specific language governing permissions and # limitations under the License. + boilerplate2024: |- + # Copyright 2024 Agnostiq Inc. + # + # This file is part of Covalent. + # + # Licensed under the Apache License 2.0 (the "License"). A copy of the + # License may be obtained with this software package or at + # + # https://www.apache.org/licenses/LICENSE-2.0 + # + # Use of this file is prohibited except in compliance with the License. + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + run: | for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" ]] ; then + if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2024" ]] ; then printf "Boilerplate is missing from $file.\n" printf "The first 15 lines of $file are\n\n" cat $file | tr -d '\r' | cat -ET | head -n 15 diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml index 30198b6be..4502b183f 100644 --- a/.github/workflows/requirements.yml +++ b/.github/workflows/requirements.yml @@ -50,10 +50,14 @@ jobs: --ignore-module=pkg_resources --ignore-module=covalent/_dispatcher_plugins --ignore-module=covalent/_shared_files + --ignore-file=covalent/quantum/** + --ignore-file=covalent/_workflow/q* + --ignore-file=covalent/_shared_files/q* + --ignore-file=covalent/_results_manager/q* + --ignore-file=covalent/_shared_files/pickling.py --ignore-file=covalent/executor/** --ignore-file=covalent/triggers/** --ignore-file=covalent/cloud_resource_manager/** - --ignore-file=covalent/quantum/qserver/** --ignore-file=covalent/_programmatic/** covalent diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 862c65795..80c7549b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,6 +90,11 @@ jobs: sdk: - 'covalent/**' - 'tests/covalent_tests/**' + qelectron: + - 'covalent/executor/quantum_plugins/**' + - 'covalent/executor/qbase.py' + - 'covalent/quantum/**' + - 'tests/qelectron_tests/**' dispatcher: - 'covalent_dispatcher/**' - 'tests/covalent_dispatcher_tests/**' @@ -134,6 +139,7 @@ jobs: echo "NEED_PYTHON=$NEED_PYTHON" >> $GITHUB_ENV echo "NEED_FRONTEND=$NEED_FRONTEND" >> $GITHUB_ENV echo "BUILD_AND_RUN_ALL=$BUILD_AND_RUN_ALL" >> $GITHUB_ENV + echo "COVALENT_DISABLE_QELECTRON_TESTS=true" >> $GITHUB_ENV - name: Set up Python if: > @@ -159,6 +165,7 @@ jobs: run: | pip install --no-cache-dir -r ./requirements.txt pip install --no-cache-dir -r ./tests/requirements.txt + pip install --no-cache-dir -r ./requirements-qelectron.txt - name: Set up Node if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL @@ -252,6 +259,18 @@ jobs: if: steps.sdk-tests.outcome == 'success' run: coverage xml -o sdk_coverage.xml + - name: Run Qelectron tests and measure coverage + id: qelectron-tests + if: > + (steps.modified-files.outputs.qelectron == 'true' + || env.BUILD_AND_RUN_ALL) && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true' + run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/qelectron_tests/core_tests --cov=covalent_qelectron --cov-config=.coveragerc + + - name: Generate Qelectron coverage report + id: qelectron-coverage + if: steps.qelectron-tests.outcome == 'success' && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true' + run: coverage xml -o qelectron_coverage.xml + - name: Run dispatcher tests and measure coverage id: dispatcher-tests if: > diff --git a/.gitignore b/.gitignore index 4e5d125f3..9e53722e0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ !pyproject.toml !requirements.txt !requirements-client.txt +!requirements-qelectron.txt !setup.py # Allow markdown etc diff --git a/CHANGELOG.md b/CHANGELOG.md index 9da4565b0..41ad2a0d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - When pushing to `master`, now the version numbers of `develop` and `master` will be compared in `man_1_push_to_master`. - Upgraded checkout action to v4 in `release.yml`. +### Added + +- Added `pennylane` as a requirement in tests due to the tutorials using it + +### Changed + +- Updated RTD notebooks to fix their behavior +- Changed the error being shown when drawing the transport graph of a lattice to a debug message instead +- Revamped README +- Reorganized `qelectron` tests +- Made qelectron an opt-in feature using `covalent[quantum]` extra + +### Removed + +- Removed unused file transfer how to guides +- Removed `pennylane` as a requirement from notebooks' requirements.txt as it comes with `covalent` +- Removed `validate_args` and `validate_region` method from `deploy_group` CLI as they were specific to AWS + +### Docs + +- Added voice cloning tutorial + +### Fixed + +- Fixed the scenario where any deploy commands would fail if the user had a non deploy compatible plugin installed +- Fixed the SQLAlchemy warning that used to show up at every fresh server start +- Fixed deploy commands' default value of plugins not being propagated to the tfvars file + +### Operations + +- Added qelectron tests to the `tests` workflow + +## [0.233.0-rc.0] - 2024-01-07 + +### Authors + +- Andrew S. Rosen +- Casey Jao +- Sankalp Sanand +- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> +- ArunPsiog <106462226+ArunPsiog@users.noreply.github.com> +- Co-authored-by: Ara Ghukasyan + + ### Added - Added feature to use custom python files as modules to be used in the electron function @@ -37,8 +81,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Reduced number of assets to upload when submitting a dispatch. -- Fixed the loading of text based assets - with a try except. -- Improved the fix of text based assets. +- Handled RecursionError on get results for a long running workflow. +- Fixed functional tests. ### Operations diff --git a/MANIFEST.in b/MANIFEST.in index 8d2699c53..71deeab82 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include VERSION include requirements.txt include requirements-client.txt +include requirements-qelectron.txt include covalent/py.typed recursive-include covalent/executor/ * recursive-include covalent_dispatcher/_service/ * diff --git a/README.md b/README.md index 85cf21d1b..780e10bd5 100644 --- a/README.md +++ b/README.md @@ -1,141 +1,202 @@ -  - +
+ hero +
+
- +[![version](https://img.shields.io/github/v/tag/AgnostiqHQ/covalent?color=%235552FF&include_prereleases&label=version&sort=semver)](https://github.com/AgnostiqHQ/covalent/blob/develop/CHANGELOG.md) +[![Static Badge](https://img.shields.io/badge/python-3.8_%7C_3.9_%7C_3.10-%235552FF)](#) +[![Static Badge](https://img.shields.io/badge/tests-passing-%235552FF?logo=github)](https://github.com/AgnostiqHQ/covalent/actions/workflows/tests.yml) +[![Static Badge](https://img.shields.io/badge/docs-passing-%235552FF)](https://docs.covalent.xyz/docs/) +[![Static Badge](https://img.shields.io/badge/codecov-88%25-%235552FF?logo=codecov)](https://codecov.io/gh/AgnostiqHQ/covalent) +[![apache](https://img.shields.io/badge/License-Apache_License_2.0-blue?color=%235552FF)](https://www.apache.org/licenses/LICENSE-2.0)
+ +
Run AI, ML, and Scientific Research Code on Any Cloud or On-Prem Cluster with a Single Line
+
+
+divider +   +divider +   +divider +   +divider +
+
-[![version](https://img.shields.io/github/v/tag/AgnostiqHQ/covalent?color=navy&include_prereleases&label=version&sort=semver)](https://github.com/AgnostiqHQ/covalent/blob/develop/CHANGELOG.md) -[![python](https://img.shields.io/pypi/pyversions/cova)](https://github.com/AgnostiqHQ/covalent) -[![tests](https://github.com/AgnostiqHQ/covalent/actions/workflows/tests.yml/badge.svg)](https://github.com/AgnostiqHQ/covalent/actions/workflows/tests.yml) -[![docs](https://readthedocs.org/projects/covalent/badge/?version=latest)](https://covalent.readthedocs.io/en/latest/?badge=latest) -[![codecov](https://codecov.io/gh/AgnostiqHQ/covalent/branch/master/graph/badge.svg?token=YGHCB3DE4P)](https://codecov.io/gh/AgnostiqHQ/covalent) -[![apache](https://img.shields.io/badge/License-Apache_License_2.0-blue)](https://www.apache.org/licenses/LICENSE-2.0) - +```bash +pip install covalent --upgrade +``` +Check our [Quick Start Guide](https://docs.covalent.xyz/docs/get-started/quick-start/) for setup instructions or dive into your [First Experiment](https://docs.covalent.xyz/docs/get-started/first-experiment/). Learn more on the [Concepts](https://docs.covalent.xyz/docs/user-documentation/concepts/concepts-index/). +### What is Covalent? +Covalent is a Python library for AI/ML engineers, developers, and researchers. It provides a straightforward approach to running compute jobs, like LLMs, generative AI, and scientific research, on various cloud platforms or on-prem clusters. -## 🤔 What is Covalent? -Covalent is a Pythonic workflow tool for computational scientists, AI/ML software engineers, and anyone who needs to run experiments on limited or expensive computing resources including quantum computers, HPC clusters, GPU arrays, and cloud services. +
+Run Code Anywhere: Execute Python functions in any cloud or on-prem cluster by changing just a single line of code. +
-Covalent enables a researcher to run computation tasks on an advanced hardware platform – such as a quantum computer or serverless HPC cluster – using a single line of code. -

-Covalent Executors -

+It is as simple as swapping the decorator with our executor plugins. Choose from [existing plugins](https://docs.covalent.xyz/docs/plugin) or [create custom ones](https://github.com/AgnostiqHQ/covalent-executor-template) for tailored interactions with any infrastructure. +
+
-## 💭 Why Covalent? +
+Abstraction of Infrastructure Management: Abstract the complexities of cloud consoles, terraform, or IaC in the background. +
+
+
-Covalent overcomes computational and operational challenges inherent in AI/ML experimentation. +
+Serverless Infrastructure: Automatically converts any infrastructure, including on-prem SLURM clusters or cloud compute, into a serverless setup. +
+
+
-| **Computational challenges** | **Operational challenges** | -| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -|
  • Advanced compute hardware is expensive, and access is often limited – shared with other researchers, for example.
  • You'd like to iterate quickly, but running large models takes time.
  • Parallel computation speeds execution, but requires careful attention to data relationships.
|
  • Proliferation of models, datasets, and hardware trials.
  • Switching between development tools, including notebooks, scripts, and submission queues.
  • Tracking, repeating, and sharing results.
| -

-Covalent value -

+If you find Covalent useful or interesting, feel free to give us a ⭐ on GitHub! Your support helps us to continue developing and improving this framework. -With Covalent, you: -- Assign functions to appropriate resources: Use advanced hardware (quantum computers, HPC clusters) for the heavy lifting and commodity hardware for bookkeeping. -- Test functions on local servers before shipping them to advanced hardware. -- Let Covalent's services analyze functions for data independence and automatically parallelize them. -- Run experiments from a Jupyter notebook (or whatever your preferred interactive Python environment is). -- Track workflows and examine results in a browser-based GUI. +
+ + + + + + + + + +
For AI/ML Practitioners and DevelopersFor Researchers
+
    +
  • Robust Compute Backend: Ideal as a backend compute framework for AI/ML applications, Large Language Models (LLMs), Generative AI, and more.
    • +
  • Cloud-Agnostic Execution: Execute high-compute tasks seamlessly across different cloud environments.
    • +
  • Infrastructure Abstraction: Directly use computing resources while keeping your business code independent from the infrastructure/resource definitions.
    • +
+ 		+
    +
  • Local-Like Access: Effortlessly connect to compute resources from your laptop, eliminating the need for SSH or complex scripts.
    • +
  • Unified Interface Across Environments: Consistent experience with on-prem HPC clusters and cloud platforms like SLURM, PBS, LSF, AWS, GCP, Azure.
    • +
  • Real-Time Monitoring Monitoring: User-friendly UI for real-time monitoring, enabling cost-effective and iterative R&D.
    • +
+
+### Out-of-box observability - [Try out the demo](http://demo.covalent.xyz/) -## 📦 Installation +If you find Covalent useful or interesting, feel free to give us a ⭐ on GitHub! Your support helps us to continue developing and improving this framework. -Covalent is developed using Python version 3.8 on Linux and macOS. The easiest way to install Covalent is using the PyPI package manager: + +
+ video +
-```console -pip install covalent -``` -Refer to the [Quick Start](https://docs.covalent.xyz/docs/get-started/quick-start) guide for quick setup instructions, or to the [First Experiment](https://docs.covalent.xyz/docs/get-started/first-experiment) guide for a more thorough approach. For a full list of supported platforms, see the Covalent [compatibility matrix](https://docs.covalent.xyz/docs/user-documentation/compatibility). -## 🚀 Getting started +### Explore Covalent Through Examples -Ready to try it? Go to the [First Experiment](https://docs.covalent.xyz/docs/get-started/first-experiment) guide in the documentation. +Jump right into practical examples to see Covalent in action. These tutorials cover a range of applications, giving you a hands-on experience: -For a more in-depth description of Covalent's features and how they work, see the [Concepts](https://docs.covalent.xyz/docs/user-documentation/concepts/concepts-index) page in the documentation. +
+ + AI Tutorial + + + MNIST Tutorial + + + Quantum Tutorial + +
- - - - - - - - - -
📚 Know more !                                                        ✍️ Tutorials and Examples                                                        
- - - [What is Covalent?](https://www.covalent.xyz/what-is-covalent/) - - [Covalent in the era of cloud-HPC](https://www.covalent.xyz/navigating-the-modern-hpc-landscape/) - - [Basic Concepts of Covalent](https://docs.covalent.xyz/docs/user-documentation/concepts/covalent-basics) - - [How does Covalent work?](#how-does-it-work) - - - - [Covalent with pytorch for classical machine learning](https://docs.covalent.xyz/docs/user-documentation/tutorials/mnist) - - [Covalent with pennylane for quantum machine learning](https://docs.covalent.xyz/docs/user-documentation/tutorials/quantumembedding) - - [Covalent with Qiskit for quantum-SVM](https://docs.covalent.xyz/docs/user-documentation/tutorials/svm/) - - [Covalent with Dask for scaling Ensemble classification](https://docs.covalent.xyz/docs/user-documentation/tutorials/ensemble/) - - [Covalent for Deep Neural Network on AWS](https://docs.covalent.xyz/docs/user-documentation/tutorials/dnn_comparison/) -
+### Explore Our Extensive Plugin Ecosystem +Covalent integrates seamlessly with a variety of platforms. Discover our range of plugins to enhance your Covalent experience: -## How Does It Work? -Covalent Architecture -Covalent has three main components: +
+
+divider +divider +divider +divider +
+
divider +divider +divider +divider
-- A Python module containing an API that you use to build manageable workflows out of new or existing Python functions. -- A set of services that run locally or on a server to dispatch and execute workflow tasks. -- A browser-based UI from which to manage workflows and view results. +### Key Features at a Glance -You compose workflows using the Covalent API by simply adding a single line of python decorator and submit them to the Covalent server. The server analyzes the workflow to determine dependencies between tasks, then dispatches each task to its specified execution backend. Independent tasks are executed concurrently if resources are available. +Get a quick overview of what Covalent offers. Our infographic summarizes the main features, providing you with a snapshot of our capabilities: -The Covalent UI displays the progress of each workflow at the level of individual tasks. +
+
+ development +
-
-

The Covalent API

-The Covalent API is a Python module containing a small collection of classes that implement server-based workflow management. The key elements are two decorators that wrap functions to create managed *tasks* and *workflows*. +
-The task decorator is called an *electron*. The electron decorator simply turns the function into a dispatchable task. +--- -The workflow decorator is called a *lattice*. The lattice decorator turns a function composed of electrons into a manageable workflow. -
-
-

Covalent Services

-The Covalent server is a lightweight service that runs on your local machine or a server. A dispatcher analyzes workflows (lattices) and hands its component functions (electrons) off to executors. Each executor is an adaptor to a backend hardware resource. Covalent has a growing list of turn-key executors for common compute backends. If no executor exists yet for your compute platform, Covalent supports writing your own. -
-
-

The Covalent GUI

-The Covalent user interface runs as a web server on the machine where the Covalent server is running. The GUI dashboard shows a list of dispatched workflows. From there, you can drill down to workflow details or a graphical view of the workflow. You can also view logs, settings, and result sets. -
+### Know More About Covalent + +For a more in-depth description of Covalent's features and how they work, see the [Concepts](https://docs.covalent.xyz/docs/user-documentation/concepts/concepts-index/) page in the documentation. -## 📚 Documentation +
-The official documentation includes tips on getting started, high-level concepts, tutorials, and the API documentation, and more. To learn more, see the [Covalent documentation](https://docs.covalent.xyz/docs). +
+divider +divider +divider +divider +
-## Troubleshooting +
-Solutions to common issues can be found in the [Troubleshooting Guide](https://docs.covalent.xyz/docs/user-documentation/troubleshooting). +### Installation + +Covalent is developed using Python on Linux and macOS. The easiest way to install Covalent is by using the PyPI package manager. + +``` +pip install covalent --upgrade +``` + +For other methods of installation, please [check the docs.](https://docs.covalent.xyz/docs/get-started/install/) + +**Deployments** + +
+Covalent offers flexible deployment options, from Docker image/AMIs for self-hosting to pip package for local installations, accommodating various use cases +
+ +
+ +
+ +divider +divider +divider + +
+ +
+
-## ✔️ Contributing +### Contributing -To contribute to Covalent, refer to the [Contribution Guidelines](https://github.com/AgnostiqHQ/covalent/blob/master/CONTRIBUTING.md). We use GitHub's [issue tracking](https://github.com/AgnostiqHQ/covalent/issues) to manage known issues, bugs, and pull requests. Get started by forking the develop branch and submitting a pull request with your contributions. Improvements to the documentation, including tutorials and how-to guides, are also welcome from the community. For more more information on adding tutorials, check the [Tutorial Guidelines](https://github.com/AgnostiqHQ/covalent/blob/master/doc/TUTORIAL_GUIDELINES.md) Participation in the Covalent community is governed by the [Code of Conduct](https://github.com/AgnostiqHQ/covalent/blob/master/CODE_OF_CONDUCT.md). + -## ⚓ Citation +To contribute to Covalent, refer to the [Contribution Guidelines](https://github.com/AgnostiqHQ/covalent/blob/master/CONTRIBUTING.md). We use GitHub's [issue tracking](https://github.com/AgnostiqHQ/covalent/issues) to manage known issues, bugs, and pull requests. Get started by forking the `develop` branch and submitting a pull request with your contributions. Improvements to the documentation, including tutorials and how-to guides, are also welcome from the community. For more information on adding tutorials, check the [Tutorial Guidelines](https://github.com/AgnostiqHQ/covalent/blob/master/doc/TUTORIAL_GUIDELINES.md). Participation in the Covalent community is governed by the [Code of Conduct](https://github.com/AgnostiqHQ/covalent/blob/master/CODE_OF_CONDUCT.md). -Please use the following citation in any publications: +### Citation -> https://doi.org/10.5281/zenodo.5903364 +Please use the following citation in any publications. -## 📃 License +[https://doi.org/10.5281/zenodo.5903364](https://zenodo.org/records/8369670) -Covalent is licensed under the Apache 2.0 License. See the [LICENSE](https://github.com/AgnostiqHQ/covalent/blob/master/LICENSE) file or contact the [support team](mailto:support@agnostiq.ai) for more details. +### License +Covalent is licensed under the Apache 2.0 License. See the [LICENSE](https://github.com/AgnostiqHQ/covalent/blob/master/LICENSE) file or contact the [support team](mailto:support@aqnostic.ai) for more details. ->For a detailed history of changes and new features, see the [Changelog](https://github.com/AgnostiqHQ/covalent/blob/master/CHANGELOG.md). +For a detailed history of changes and new features, see the [Changelog](https://github.com/AgnostiqHQ/covalent/blob/master/CHANGELOG.md). diff --git a/VERSION b/VERSION index 48c3454f0..3379c667f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.233.2-rc.0 +0.233.0-rc.0 diff --git a/covalent/__init__.py b/covalent/__init__.py index 6d88af00d..0cfc7bbe7 100644 --- a/covalent/__init__.py +++ b/covalent/__init__.py @@ -16,6 +16,7 @@ """Main Covalent public functionality.""" +import contextlib from importlib import metadata from . import _file_transfer as fs # nopycln: import @@ -48,9 +49,11 @@ lattice, ) from ._workflow.electron import wait # nopycln: import -from ._workflow.qelectron import qelectron # nopycln: import from .executor.utils import get_context # nopycln: import -from .quantum import QCluster # nopycln: import + +with contextlib.suppress(ImportError): + from ._workflow.qelectron import qelectron # nopycln: import + from .quantum import QCluster # nopycln: import __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/covalent/_file_transfer/strategies/gcloud_strategy.py b/covalent/_file_transfer/strategies/gcloud_strategy.py index 22884d73b..24c3e4e51 100644 --- a/covalent/_file_transfer/strategies/gcloud_strategy.py +++ b/covalent/_file_transfer/strategies/gcloud_strategy.py @@ -41,7 +41,7 @@ class GCloud(FileTransferStrategy): def __init__(self, credentials: str = None, project_id: str = None): if credentials is not None: - credentials_json = Path(credentials).resolve() + credentials_json = Path(credentials).expanduser().resolve() if not credentials_json.is_file(): raise ValueError("Cloud not locate credentials for GCloud file transfers.") diff --git a/covalent/_results_manager/results_manager.py b/covalent/_results_manager/results_manager.py index e31670c90..4c751206a 100644 --- a/covalent/_results_manager/results_manager.py +++ b/covalent/_results_manager/results_manager.py @@ -496,15 +496,23 @@ def get_result( The Result object from the Covalent server """ + max_attempts = int(os.getenv("COVALENT_GET_RESULT_RETRIES", 10)) + num_attempts = 0 + while num_attempts < max_attempts: + try: + return _get_result_multistage( + dispatch_id=dispatch_id, + wait=wait, + dispatcher_addr=dispatcher_addr, + status_only=status_only, + results_dir=results_dir, + workflow_output=workflow_output, + intermediate_outputs=intermediate_outputs, + sublattice_results=sublattice_results, + qelectron_db=qelectron_db, + ) - return _get_result_multistage( - dispatch_id=dispatch_id, - wait=wait, - dispatcher_addr=dispatcher_addr, - status_only=status_only, - results_dir=results_dir, - workflow_output=workflow_output, - intermediate_outputs=intermediate_outputs, - sublattice_results=sublattice_results, - qelectron_db=qelectron_db, - ) + except RecursionError as re: + app_log.error(re) + num_attempts += 1 + raise RuntimeError("Timed out waiting for result. Please retry or check dispatch.") diff --git a/covalent/_shared_files/qelectron_utils.py b/covalent/_shared_files/qelectron_utils.py index 1c0a8c6f3..387722a2a 100644 --- a/covalent/_shared_files/qelectron_utils.py +++ b/covalent/_shared_files/qelectron_utils.py @@ -14,10 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib +import inspect +from typing import Any, Tuple -from covalent.quantum.qserver.database import Database +import cloudpickle +from pennylane._device import Device from .logger import app_log +from .pickling import _qml_mods_pickle + +_IMPORT_PATH_SEPARATOR = ":" def get_qelectron_db_path(dispatch_id: str, task_id: int): @@ -28,6 +35,8 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int): AS WHERE THE USER'S TASK FUNCTION IS BEING RUN. """ + from covalent.quantum.qserver.database import Database + database = Database() db_path = database.get_db_path(dispatch_id=dispatch_id, node_id=task_id) @@ -38,3 +47,50 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int): else: app_log.debug(f"Qelectron database not found for task {task_id}") return None + + +@_qml_mods_pickle +def cloudpickle_serialize(obj): + return cloudpickle.dumps(obj) + + +def cloudpickle_deserialize(obj): + return cloudpickle.loads(obj) + + +def select_first_executor(qnode, executors): + """Selects the first executor to run the qnode""" + return executors[0] + + +def get_import_path(obj) -> Tuple[str, str]: + """ + Determine the import path of an object. + """ + if module := inspect.getmodule(obj): + module_path = module.__name__ + class_name = obj.__name__ + return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}" + raise RuntimeError(f"Unable to determine import path for {obj}.") + + +def import_from_path(path: str) -> Any: + """ + Import a class from a path. + """ + module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR) + module = importlib.import_module(module_path) + return getattr(module, class_name) + + +def get_original_shots(dev: Device): + """ + Recreate vector of shots if device has a shot vector. + """ + if not dev.shot_vector: + return dev.shots + + shot_sequence = [] + for shots in dev.shot_vector: + shot_sequence.extend([shots.shots] * shots.copies) + return type(dev.shot_vector)(shot_sequence) diff --git a/covalent/_shared_files/qresult_utils.py b/covalent/_shared_files/qresult_utils.py index 9ffe3c182..efad64797 100644 --- a/covalent/_shared_files/qresult_utils.py +++ b/covalent/_shared_files/qresult_utils.py @@ -23,7 +23,7 @@ from pennylane.tape import QuantumTape from .._workflow.qdevice import QEDevice -from .utils import get_original_shots +from .qelectron_utils import get_original_shots def re_execute( diff --git a/covalent/_shared_files/utils.py b/covalent/_shared_files/utils.py index e7bd60368..f41899f24 100644 --- a/covalent/_shared_files/utils.py +++ b/covalent/_shared_files/utils.py @@ -16,19 +16,14 @@ """General utils for Covalent.""" -import importlib import inspect import shutil import socket from datetime import timedelta -from typing import Any, Callable, Dict, List, Tuple - -import cloudpickle -from pennylane._device import Device +from typing import Callable, Dict, List, Tuple from . import logger from .config import get_config -from .pickling import _qml_mods_pickle app_log = logger.app_log log_stack_info = logger.log_stack_info @@ -37,9 +32,6 @@ DEFAULT_UI_PORT = get_config("user_interface.port") -_IMPORT_PATH_SEPARATOR = ":" - - def get_ui_url(path): baseUrl = f"http://{DEFAULT_UI_ADDRESS}:{DEFAULT_UI_PORT}" return f"{baseUrl}{path}" @@ -264,49 +256,16 @@ def copy_file_locally(src_uri, dest_uri): shutil.copyfile(src_path, dest_path) -@_qml_mods_pickle -def cloudpickle_serialize(obj): - return cloudpickle.dumps(obj) - - -def cloudpickle_deserialize(obj): - return cloudpickle.loads(obj) - - -def select_first_executor(qnode, executors): - """Selects the first executor to run the qnode""" - return executors[0] - - -def get_import_path(obj) -> Tuple[str, str]: - """ - Determine the import path of an object. +def get_qelectron_db_path(dispatch_id: str, task_id: int): """ - module = inspect.getmodule(obj) - if module: - module_path = module.__name__ - class_name = obj.__name__ - return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}" - raise RuntimeError(f"Unable to determine import path for {obj}.") + Return the path to the Qelectron database for a given dispatch_id and task_id. - -def import_from_path(path: str) -> Any: - """ - Import a class from a path. + This is a proxy to qelectron_utils.get_qelectron_db_path() for removing qelectron dependency. """ - module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR) - module = importlib.import_module(module_path) - return getattr(module, class_name) + try: + from .qelectron_utils import get_qelectron_db_path -def get_original_shots(dev: Device): - """ - Recreate vector of shots if device has a shot vector. - """ - if not dev.shot_vector: - return dev.shots - - shot_sequence = [] - for shots in dev.shot_vector: - shot_sequence.extend([shots.shots] * shots.copies) - return type(dev.shot_vector)(shot_sequence) + return get_qelectron_db_path(dispatch_id, task_id) + except ImportError: + return None diff --git a/covalent/_workflow/qelectron.py b/covalent/_workflow/qelectron.py index e0356723a..641e4a78e 100644 --- a/covalent/_workflow/qelectron.py +++ b/covalent/_workflow/qelectron.py @@ -19,7 +19,7 @@ import pennylane as qml -from .._shared_files.utils import get_import_path, get_original_shots +from .._shared_files.qelectron_utils import get_import_path, get_original_shots from ..quantum.qcluster import QCluster from ..quantum.qcluster.base import AsyncBaseQCluster, BaseQExecutor from ..quantum.qcluster.simulator import Simulator diff --git a/covalent/_workflow/qnode.py b/covalent/_workflow/qnode.py index 7e97fee34..f2019aaa0 100644 --- a/covalent/_workflow/qnode.py +++ b/covalent/_workflow/qnode.py @@ -25,9 +25,9 @@ from .._results_manager.qresult import QNodeFutureResult from .._shared_files import logger +from .._shared_files.qelectron_utils import get_original_shots from .._shared_files.qinfo import QElectronInfo, QNodeSpecs from .._shared_files.qresult_utils import re_execute -from .._shared_files.utils import get_original_shots from ..executor.qbase import BaseQExecutor from .qdevice import QEDevice diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py index 4721fb70c..974e2f61c 100644 --- a/covalent/cloud_resource_manager/core.py +++ b/covalent/cloud_resource_manager/core.py @@ -425,6 +425,17 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None: # Setup terraform infra variables as passed by the user tf_vars_env_dict = os.environ.copy() + # Write the default values to the terraform.tfvars file + infra_settings = self.ExecutorInfraDefaults.schema()["properties"] + with open(tfvars_file, "w", encoding="utf-8") as f: + for key, value in infra_settings.items(): + if "default" in value: + tf_vars_env_dict[f"TF_VAR_{key}"] = value["default"] + + if value["default"] != "": + f.write(f'{key}="{value["default"]}"\n') + + # Overwrite the default values with the user passed values if self.executor_options: with open(tfvars_file, "w", encoding="utf-8") as f: for key, value in self.executor_options.items(): diff --git a/covalent/executor/__init__.py b/covalent/executor/__init__.py index ffc139f96..be68b1182 100644 --- a/covalent/executor/__init__.py +++ b/covalent/executor/__init__.py @@ -31,7 +31,6 @@ from .._shared_files import logger from .._shared_files.config import get_config, update_config -from ..quantum import QCluster, Simulator from .base import BaseExecutor app_log = logger.app_log @@ -284,6 +283,8 @@ class _QExecutorManager: """ def __init__(self): + from ..quantum import QCluster, Simulator + # Dictionary mapping executor name to executor class self.executor_plugins_map: Dict[str, Any] = { "QCluster": QCluster, @@ -370,11 +371,12 @@ def validate_module(self, module_obj) -> None: _executor_manager = _ExecutorManager() -_qexecutor_manager = _QExecutorManager() - for name in _executor_manager.executor_plugins_map: plugin_class = _executor_manager.executor_plugins_map[name] globals()[plugin_class.__name__] = plugin_class -for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values(): - globals()[qexecutor_cls.__name__] = qexecutor_cls +# Only creating the qexecutor manager if its requirements are installed +with contextlib.suppress(ImportError): + _qexecutor_manager = _QExecutorManager() + for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values(): + globals()[qexecutor_cls.__name__] = qexecutor_cls diff --git a/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py b/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py index 43531ddef..a959122e3 100644 --- a/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py +++ b/covalent/executor/quantum_plugins/qiskit_plugin/qiskit_plugin.py @@ -25,7 +25,7 @@ from runtime_sampler import QiskitRuntimeSampler from covalent._shared_files.config import get_config -from covalent._shared_files.utils import import_from_path +from covalent._shared_files.qelectron_utils import import_from_path from covalent.executor.qbase import ( AsyncBaseQExecutor, BaseThreadPoolQExecutor, diff --git a/covalent/executor/utils/wrappers.py b/covalent/executor/utils/wrappers.py index 4cbd7fccb..8f7fd8256 100644 --- a/covalent/executor/utils/wrappers.py +++ b/covalent/executor/utils/wrappers.py @@ -29,7 +29,7 @@ import requests -from covalent._shared_files.qelectron_utils import get_qelectron_db_path +from covalent._shared_files.utils import get_qelectron_db_path from covalent._workflow.depsbash import DepsBash from covalent._workflow.depscall import RESERVED_RETVAL_KEY__FILES, DepsCall from covalent._workflow.depspip import DepsPip diff --git a/covalent/quantum/qclient/local_client.py b/covalent/quantum/qclient/local_client.py index 89bff7382..8ca46c99d 100644 --- a/covalent/quantum/qclient/local_client.py +++ b/covalent/quantum/qclient/local_client.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..._shared_files.utils import cloudpickle_deserialize, cloudpickle_serialize +from ..._shared_files.qelectron_utils import cloudpickle_deserialize, cloudpickle_serialize from ..qserver import LocalQServer from .base_client import BaseQClient diff --git a/covalent/quantum/qcluster/clusters.py b/covalent/quantum/qcluster/clusters.py index fe56aecbf..7aa1de314 100644 --- a/covalent/quantum/qcluster/clusters.py +++ b/covalent/quantum/qcluster/clusters.py @@ -17,7 +17,7 @@ import base64 from typing import Callable, Union -from ..._shared_files.utils import cloudpickle_deserialize, cloudpickle_serialize +from ..._shared_files.qelectron_utils import cloudpickle_deserialize, cloudpickle_serialize from .base import AsyncBaseQCluster, BaseQExecutor from .default_selectors import selector_map diff --git a/covalent/quantum/qserver/core.py b/covalent/quantum/qserver/core.py index f8efe09d8..2bc31c913 100644 --- a/covalent/quantum/qserver/core.py +++ b/covalent/quantum/qserver/core.py @@ -25,12 +25,12 @@ from pennylane.tape import QuantumScript -from ..._shared_files.qinfo import QElectronInfo, QNodeSpecs -from ..._shared_files.utils import ( +from ..._shared_files.qelectron_utils import ( cloudpickle_deserialize, cloudpickle_serialize, select_first_executor, ) +from ..._shared_files.qinfo import QElectronInfo, QNodeSpecs from ...executor.utils import get_context from ..qcluster.base import AsyncBaseQCluster, BaseQExecutor from .database import Database diff --git a/covalent_dispatcher/_cli/groups/deploy_group.py b/covalent_dispatcher/_cli/groups/deploy_group.py index a9510c09f..ff8c06a23 100644 --- a/covalent_dispatcher/_cli/groups/deploy_group.py +++ b/covalent_dispatcher/_cli/groups/deploy_group.py @@ -24,7 +24,6 @@ from pathlib import Path from typing import Callable, Dict, Tuple -import boto3 import click from rich.console import Console from rich.table import Table @@ -176,12 +175,20 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool) $ covalent deploy up awslambda --verbose --region=us-east-1 --instance-type=t2.micro """ + cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)} - if msg := validate_args(cmd_options): - # Message is not None, so there was an error. - click.echo(msg) + + try: + crm = get_crm_object(executor_name, cmd_options) + except (KeyError, AttributeError): + click.echo( + click.style( + f"Warning: '{executor_name}' is not a valid executor for deployment.", + fg="yellow", + ) + ) sys.exit(1) - crm = get_crm_object(executor_name, cmd_options) + if help: click.echo(Console().print(get_up_help_table(crm))) sys.exit(0) @@ -212,7 +219,18 @@ def down(executor_name: str, verbose: bool) -> None: $ covalent deploy down ecs --verbose """ - crm = get_crm_object(executor_name) + + try: + crm = get_crm_object(executor_name) + except (KeyError, AttributeError): + click.echo( + click.style( + f"Warning: '{executor_name}' is not a valid executor for deployment.", + fg="yellow", + ) + ) + sys.exit(1) + _command = partial(crm.down) _run_command_and_show_output(_command, "Destroying resources...", verbose=verbose) @@ -247,7 +265,7 @@ def status(executor_names: Tuple[str]) -> None: for name in _executor_manager.executor_plugins_map if name not in ["dask", "local", "remote_executor"] ] - click.echo(f"Executors: {', '.join(executor_names)}") + click.echo(f"Installed executors: {', '.join(executor_names)}") table = Table() table.add_column("Executor", justify="center") @@ -260,7 +278,9 @@ def status(executor_names: Tuple[str]) -> None: crm = get_crm_object(executor_name) crm_status = crm.status() table.add_row(executor_name, crm_status, description[crm_status]) - except KeyError: + except (KeyError, AttributeError): + # Added the AttributeError here as well in case the executor does not + # have the ExecutorPluginDefaults or ExecutorInfraDefaults classes. invalid_executor_names.append(executor_name) click.echo(Console().print(table)) @@ -268,23 +288,7 @@ def status(executor_names: Tuple[str]) -> None: if invalid_executor_names: click.echo( click.style( - f"Warning: {', '.join(invalid_executor_names)} are not valid executors.", + f"Warning: Invalid executors for deployment -> '{', '.join(invalid_executor_names)}'", fg="yellow", ) ) - - -def validate_args(args: dict): - message = None - if len(args) == 0: - return message - if "region" in args and args["region"] != "": - if not validate_region(args["region"]): - return f"Unable to find the provided region: {args['region']}" - - -def validate_region(region_name: str): - ec2_client = boto3.client("ec2") - response = ec2_client.describe_regions() - exists = region_name in [item["RegionName"] for item in response["Regions"]] - return exists diff --git a/covalent_dispatcher/_cli/service.py b/covalent_dispatcher/_cli/service.py index faa14617d..df2299a8f 100644 --- a/covalent_dispatcher/_cli/service.py +++ b/covalent_dispatcher/_cli/service.py @@ -27,6 +27,7 @@ import sys import time import traceback +import warnings from pathlib import Path from subprocess import DEVNULL, Popen from typing import Optional @@ -50,6 +51,7 @@ from rich.syntax import Syntax from rich.table import Table from rich.text import Text +from sqlalchemy import exc as sa_exc from covalent._shared_files.config import ConfigManager, get_config, reload_config, set_config @@ -67,6 +69,9 @@ ZOMBIE_PROCESS_STATUS_MSG = "Covalent server is unhealthy: Process is in zombie status" STOPPED_PROCESS_STATUS_MSG = "Covalent server is unhealthy: Process is in stopped status" +# Ignore SQLAlchemy warnings +warnings.simplefilter("ignore", category=sa_exc.SAWarning) + def print_header(console): branding_title = Text("Covalent", style="bold blue") diff --git a/covalent_dispatcher/_db/dispatchdb.py b/covalent_dispatcher/_db/dispatchdb.py index 9fcb4d2dd..621022777 100644 --- a/covalent_dispatcher/_db/dispatchdb.py +++ b/covalent_dispatcher/_db/dispatchdb.py @@ -65,6 +65,8 @@ def extract_graph_node(node): def extract_metadata(metadata: dict): + # TODO: This is an outdated method + try: # avoid mutating original metadata metadata = copy.deepcopy(metadata) @@ -81,20 +83,20 @@ def extract_metadata(metadata: dict): else: metadata["executor_name"] = f"<{executor.__class__.__name__}>" - metadata["deps"] = encode_dict(metadata["deps"]) - call_before = metadata["call_before"] - call_after = metadata["call_after"] + metadata["hooks"]["deps"] = encode_dict(metadata["hooks"]["deps"]) + call_before = metadata["hooks"]["call_before"] + call_after = metadata["hooks"]["call_after"] for i, dep in enumerate(call_before): call_before[i] = str(dep) for i, dep in enumerate(call_after): call_after[i] = str(dep) - metadata["call_before"] = call_before - metadata["call_after"] = call_after + metadata["hooks"]["call_before"] = call_before + metadata["hooks"]["call_after"] = call_after except (KeyError, AttributeError) as ex: - app_log.error(f"Exception when trying to extract metadata: {ex}") + app_log.debug(f"Exception when trying to extract metadata: {ex}") return metadata diff --git a/covalent_ui/api/v1/data_layer/electron_dal.py b/covalent_ui/api/v1/data_layer/electron_dal.py index 5f85c965a..9e10ee154 100644 --- a/covalent_ui/api/v1/data_layer/electron_dal.py +++ b/covalent_ui/api/v1/data_layer/electron_dal.py @@ -28,7 +28,6 @@ from covalent._results_manager.results_manager import get_result from covalent._shared_files import logger from covalent._shared_files.config import get_config -from covalent.quantum.qserver.database import Database from covalent_dispatcher._core.execution import _get_task_inputs as get_task_inputs from covalent_ui.api.v1.data_layer.lattice_dal import Lattices from covalent_ui.api.v1.database.schema.electron import Electron @@ -322,11 +321,17 @@ def get_electron_inputs(self, dispatch_id: uuid.UUID, electron_id: int) -> str: def _get_qelectron_db_dict(self, dispatch_id: str, node_id: int) -> dict: """Return the QElectron DB for a given node.""" - electron = self.get_electrons_id(dispatch_id, node_id) + try: + from covalent.quantum.qserver.database import Database - database = Database(electron.storage_path) - qelectron_db_dict = database.get_db_dict( - dispatch_id=dispatch_id, node_id=node_id, direct_path=True - ) + electron = self.get_electrons_id(dispatch_id, node_id) + + database = Database(electron.storage_path) + qelectron_db_dict = database.get_db_dict( + dispatch_id=dispatch_id, node_id=node_id, direct_path=True + ) - return qelectron_db_dict + return qelectron_db_dict + except ImportError: + app_log.debug("QElectron not installed.") + return {} diff --git a/doc/source/_static/abstract_infra.png b/doc/source/_static/abstract_infra.png new file mode 100644 index 000000000..dfe3aefb7 Binary files /dev/null and b/doc/source/_static/abstract_infra.png differ diff --git a/doc/source/_static/ai.svg b/doc/source/_static/ai.svg new file mode 100644 index 000000000..dbcb403dc --- /dev/null +++ b/doc/source/_static/ai.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/ai_tutorial.svg b/doc/source/_static/ai_tutorial.svg new file mode 100644 index 000000000..15916d38d --- /dev/null +++ b/doc/source/_static/ai_tutorial.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/aws.svg b/doc/source/_static/aws.svg new file mode 100644 index 000000000..6d7ab84fc --- /dev/null +++ b/doc/source/_static/aws.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/azure.svg b/doc/source/_static/azure.svg new file mode 100644 index 000000000..144bcc0e2 --- /dev/null +++ b/doc/source/_static/azure.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/banner_executor.gif b/doc/source/_static/banner_executor.gif new file mode 100644 index 000000000..05f6c5abb Binary files /dev/null and b/doc/source/_static/banner_executor.gif differ diff --git a/doc/source/_static/cloud_hosted.svg b/doc/source/_static/cloud_hosted.svg new file mode 100644 index 000000000..55138a28c --- /dev/null +++ b/doc/source/_static/cloud_hosted.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/doc/source/_static/cloud_hpc.svg b/doc/source/_static/cloud_hpc.svg new file mode 100644 index 000000000..e0a4637ba --- /dev/null +++ b/doc/source/_static/cloud_hpc.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/code_snippet.svg b/doc/source/_static/code_snippet.svg new file mode 100644 index 000000000..e17f9e330 --- /dev/null +++ b/doc/source/_static/code_snippet.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/doc/source/_static/computing.svg b/doc/source/_static/computing.svg new file mode 100644 index 000000000..a853fb2b6 --- /dev/null +++ b/doc/source/_static/computing.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/doc/source/_static/concepts.svg b/doc/source/_static/concepts.svg new file mode 100644 index 000000000..f6865871d --- /dev/null +++ b/doc/source/_static/concepts.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/concepts_of_covalent.svg b/doc/source/_static/concepts_of_covalent.svg new file mode 100644 index 000000000..68df9c1a5 --- /dev/null +++ b/doc/source/_static/concepts_of_covalent.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/copy.svg b/doc/source/_static/copy.svg new file mode 100644 index 000000000..ded49db97 --- /dev/null +++ b/doc/source/_static/copy.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/covalent_work.svg b/doc/source/_static/covalent_work.svg new file mode 100644 index 000000000..9ad4cea27 --- /dev/null +++ b/doc/source/_static/covalent_work.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/dask.svg b/doc/source/_static/dask.svg new file mode 100644 index 000000000..11d5e91bd --- /dev/null +++ b/doc/source/_static/dask.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/deployment.svg b/doc/source/_static/deployment.svg new file mode 100644 index 000000000..77f087d6c --- /dev/null +++ b/doc/source/_static/deployment.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/development.svg b/doc/source/_static/development.svg new file mode 100644 index 000000000..7dc9b7d27 --- /dev/null +++ b/doc/source/_static/development.svg @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/documentation.svg b/doc/source/_static/documentation.svg new file mode 100644 index 000000000..4bcfde9d4 --- /dev/null +++ b/doc/source/_static/documentation.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/doc/source/_static/examples.svg b/doc/source/_static/examples.svg new file mode 100644 index 000000000..b2f171038 --- /dev/null +++ b/doc/source/_static/examples.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/executor.gif b/doc/source/_static/executor.gif new file mode 100644 index 000000000..e82724d3c Binary files /dev/null and b/doc/source/_static/executor.gif differ diff --git a/doc/source/_static/executors.svg b/doc/source/_static/executors.svg new file mode 100644 index 000000000..f11e472f6 --- /dev/null +++ b/doc/source/_static/executors.svg @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/executors_ship.png b/doc/source/_static/executors_ship.png new file mode 100644 index 000000000..10f8680a3 Binary files /dev/null and b/doc/source/_static/executors_ship.png differ diff --git a/doc/source/_static/getting_started.svg b/doc/source/_static/getting_started.svg new file mode 100644 index 000000000..3a2f3a212 --- /dev/null +++ b/doc/source/_static/getting_started.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/google.svg b/doc/source/_static/google.svg new file mode 100644 index 000000000..6ef470625 --- /dev/null +++ b/doc/source/_static/google.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/high_compute.svg b/doc/source/_static/high_compute.svg new file mode 100644 index 000000000..6dff0ed22 --- /dev/null +++ b/doc/source/_static/high_compute.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/hpc.svg b/doc/source/_static/hpc.svg new file mode 100644 index 000000000..35d605858 --- /dev/null +++ b/doc/source/_static/hpc.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/doc/source/_static/ibmq.svg b/doc/source/_static/ibmq.svg new file mode 100644 index 000000000..d8169036b --- /dev/null +++ b/doc/source/_static/ibmq.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/kubernetes.svg b/doc/source/_static/kubernetes.svg new file mode 100644 index 000000000..80e3aa6b3 --- /dev/null +++ b/doc/source/_static/kubernetes.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/local-laptop.svg b/doc/source/_static/local-laptop.svg new file mode 100644 index 000000000..fb5a0f5f0 --- /dev/null +++ b/doc/source/_static/local-laptop.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/many_more.svg b/doc/source/_static/many_more.svg new file mode 100644 index 000000000..cff85a5c9 --- /dev/null +++ b/doc/source/_static/many_more.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/doc/source/_static/mnist.svg b/doc/source/_static/mnist.svg new file mode 100644 index 000000000..4615c109a --- /dev/null +++ b/doc/source/_static/mnist.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/mnist_tutorial.svg b/doc/source/_static/mnist_tutorial.svg new file mode 100644 index 000000000..6123c11b4 --- /dev/null +++ b/doc/source/_static/mnist_tutorial.svg @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/onprem_hosted.svg b/doc/source/_static/onprem_hosted.svg new file mode 100644 index 000000000..f83852369 --- /dev/null +++ b/doc/source/_static/onprem_hosted.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/orchestration.svg b/doc/source/_static/orchestration.svg new file mode 100644 index 000000000..141cdfebe --- /dev/null +++ b/doc/source/_static/orchestration.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/source/_static/quantum_tutorial.svg b/doc/source/_static/quantum_tutorial.svg new file mode 100644 index 000000000..02a4762f0 --- /dev/null +++ b/doc/source/_static/quantum_tutorial.svg @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/readmeVid-gif.gif b/doc/source/_static/readmeVid-gif.gif new file mode 100644 index 000000000..ac5d486e1 Binary files /dev/null and b/doc/source/_static/readmeVid-gif.gif differ diff --git a/doc/source/_static/readmeVid.mp4 b/doc/source/_static/readmeVid.mp4 new file mode 100644 index 000000000..2724a2681 Binary files /dev/null and b/doc/source/_static/readmeVid.mp4 differ diff --git a/doc/source/_static/readme_executor.svg b/doc/source/_static/readme_executor.svg new file mode 100644 index 000000000..5fca0a837 --- /dev/null +++ b/doc/source/_static/readme_executor.svg @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/readme_hero.svg b/doc/source/_static/readme_hero.svg new file mode 100644 index 000000000..af229fb0a --- /dev/null +++ b/doc/source/_static/readme_hero.svg @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/self.svg b/doc/source/_static/self.svg new file mode 100644 index 000000000..cbebfdace --- /dev/null +++ b/doc/source/_static/self.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/doc/source/_static/serverless-illustration.png b/doc/source/_static/serverless-illustration.png new file mode 100644 index 000000000..952688422 Binary files /dev/null and b/doc/source/_static/serverless-illustration.png differ diff --git a/doc/source/_static/slack.svg b/doc/source/_static/slack.svg new file mode 100644 index 000000000..03f4ef0a4 --- /dev/null +++ b/doc/source/_static/slack.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/doc/source/_static/slurm.svg b/doc/source/_static/slurm.svg new file mode 100644 index 000000000..700fe19a0 --- /dev/null +++ b/doc/source/_static/slurm.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/doc/source/_static/tutorial_heading.svg b/doc/source/_static/tutorial_heading.svg new file mode 100644 index 000000000..8c3784321 --- /dev/null +++ b/doc/source/_static/tutorial_heading.svg @@ -0,0 +1,46 @@ + + + +
+ +
+ Tutorials +
+
+ + diff --git a/doc/source/_static/what_is_covalent.svg b/doc/source/_static/what_is_covalent.svg new file mode 100644 index 000000000..fde5bb444 --- /dev/null +++ b/doc/source/_static/what_is_covalent.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/doc/source/_static/work.svg b/doc/source/_static/work.svg new file mode 100644 index 000000000..117caacc9 --- /dev/null +++ b/doc/source/_static/work.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/workflow.svg b/doc/source/_static/workflow.svg new file mode 100644 index 000000000..6dd6831e7 --- /dev/null +++ b/doc/source/_static/workflow.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/workflows.svg b/doc/source/_static/workflows.svg new file mode 100644 index 000000000..0266074fc --- /dev/null +++ b/doc/source/_static/workflows.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/doc/source/how_to/coding/file_transfers_for_workflows_from_remote.ipynb b/doc/source/how_to/coding/file_transfers_for_workflows_from_remote.ipynb deleted file mode 100644 index 35412ce95..000000000 --- a/doc/source/how_to/coding/file_transfers_for_workflows_from_remote.ipynb +++ /dev/null @@ -1,189 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5120ef0f-8510-4ad7-a0ea-2e306aec0064", - "metadata": { - "tags": [] - }, - "source": [ - "## Transferring Files to and from an S3 Bucket\n", - "\n", - "Retrieve a file from a remote host's filesystem before executing a workflow using Rsync via SSH.\n", - "\n", - "### Prerequisites\n", - "\n", - "1. Define the read (source) file path. \n", - "2. Create a source file to transfer." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "a20575c4", - "metadata": {}, - "outputs": [], - "source": [ - "import covalent as ct \n", - "\n", - "from pathlib import Path\n", - "\n", - "# define source & destination filepaths \n", - "source_filepath = Path('./my_source_file').resolve()\n", - "\n", - "# create an example file\n", - "source_filepath.touch()" - ] - }, - { - "cell_type": "markdown", - "id": "57700440", - "metadata": {}, - "source": [ - "### Procedure\n", - "\n", - "Transfer a file from an S3 bucket to a local filesystem using the boto3 library. \n", - "\n", - "In the following example a zip file is downloaded from an S3 bucket before electron execution. The electron processes the files, then the processed files are uploaded back to the S3 bucket.\n", - "\n", - "1. Define two Covalent `FileTransfer` objects and a Covalent `S3` strategy object:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "052ed080", - "metadata": {}, - "outputs": [], - "source": [ - "import covalent as ct\n", - "import zipfile\n", - "import os\n", - "\n", - "strategy = ct.fs_strategies.S3()\n", - "\n", - "ft_2 = ct.fs.FileTransfer('/home/ubuntu/tmp-dir/images.zip','s3://covalent-tmp/images.zip',strategy = strategy,order=ct.fs.Order.AFTER)\n", - "ft_1 = ct.fs.FileTransfer('s3://covalent-tmp/test_vids.zip','/home/ubuntu/tmp-dir/test_vids.zip',strategy = strategy)\n" - ] - }, - { - "cell_type": "markdown", - "id": "0a213bf5", - "metadata": {}, - "source": [ - "2. Define an electron to:\n", - " 1. Download a zip file from S3\n", - " 2. Unzip the file\n", - " 3. Perform some processing on the contents (omitted here as irrelevant to the demo)\n", - " 4. Zip the files\n", - " 5. Upload the zip file to S3:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "4519a791", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2023-07-25 15:52:15,882] [DEBUG] s3_strategy.py: Line 57 in download: Is dir: False\n", - "[2023-07-25 15:52:15,883] [DEBUG] s3_strategy.py: Line 66 in download: S3 download bucket: covalent-tmp, from_filepath: test_vids.zip, to_filepath /home/ubuntu/tmp-dir/test_vids.zip.\n", - "[2023-07-25 15:52:15,884] [DEBUG] s3_strategy.py: Line 133 in upload: S3 upload bucket: covalent-tmp, from_filepath: /home/ubuntu/tmp-dir/images.zip, to_filepath images.zip.\n" - ] - } - ], - "source": [ - "@ct.electron(files = [ft_1,ft_2])\n", - "def unzip_zip(files=[]):\n", - " path = \"/home/ubuntu/tmp-dir\"\n", - " # Unzip downloaded data\n", - " with zipfile.ZipFile(path + \"/test_vids.zip\", 'r') as zip_ref:\n", - " zip_ref.extractall(path)\n", - " \n", - " # Perform operations on the files\n", - " # ...\n", - " \n", - " # Zip files to upload \n", - " with zipfile.ZipFile(path + \"/images.zip\", 'w', zipfile.ZIP_DEFLATED) as ziph:\n", - " for root, dirs, files in os.walk(path + '/test_vids'):\n", - " for file in files:\n", - " ziph.write(os.path.join(root, file), \n", - " os.path.relpath(os.path.join(root, file), \n", - " os.path.join(path, '..')))" - ] - }, - { - "cell_type": "markdown", - "id": "f05a5759", - "metadata": {}, - "source": [ - "3. Create and dispatch a lattice to run the electron:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1c2d3589", - "metadata": {}, - "outputs": [], - "source": [ - "@ct.lattice\n", - "def run_electrons():\n", - " return unzip_zip()\n", - "\n", - "dispatch_id = ct.dispatch(run_electrons)()" - ] - }, - { - "cell_type": "markdown", - "id": "eea96bae", - "metadata": {}, - "source": [ - "Notes:\n", - "- This example illustrates a typical pattern in which files are downloaded from remote storage, are processed, and the results are uploaded to the same remote storage. Other scenarios can of course be implemented with the Covalent components illustrated here (`FileTransfer`, `FileTransferStrategy`, `@electron`).\n", - "- The example puts everything in one electron. For a real-world scenario of any complexity, a better practice would be to break the task into small sub-tasks, each in its own electron." - ] - }, - { - "cell_type": "markdown", - "id": "244061a2", - "metadata": {}, - "source": [ - "### See Also\n", - "\n", - "[Transferring Local Files During Workflows](./file_transfers_for_workflows_local.ipynb)\n", - "\n", - "[Transferring Remote Files After a Workflow](./file_transfers_for_workflows_to_remote.ipynb)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - }, - "vscode": { - "interpreter": { - "hash": "" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/doc/source/how_to/coding/file_transfers_for_workflows_local.ipynb b/doc/source/how_to/coding/file_transfers_for_workflows_local.ipynb index 5fd90ae97..cd265428e 100644 --- a/doc/source/how_to/coding/file_transfers_for_workflows_local.ipynb +++ b/doc/source/how_to/coding/file_transfers_for_workflows_local.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 1, "id": "5a8139da", "metadata": {}, "outputs": [], @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "id": "089aa573", "metadata": {}, "outputs": [], @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "id": "34e40034", "metadata": {}, "outputs": [], @@ -85,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "id": "279dc3fa", "metadata": {}, "outputs": [], @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "id": "de6ed992", "metadata": {}, "outputs": [], @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "id": "789a74ef", "metadata": {}, "outputs": [], @@ -157,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "id": "a30852d1-2fa4-4ffb-b01d-7e7cc9925182", "metadata": {}, "outputs": [], @@ -182,7 +182,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "id": "0a6a94bf", "metadata": {}, "outputs": [ @@ -194,22 +194,23 @@ "Lattice Result\n", "==============\n", "status: COMPLETED\n", - "result: /Users/mini-me/agnostiq/covalent/doc/source/how_to/coding/my_dest_file\n", - "input args: []\n", - "input kwargs: {}\n", + "result: /Users/sankalpsanand/dev/covalent/doc/source/how_to/coding/my_dest_file\n", + "input args: \n", + "input kwargs: \n", "error: None\n", "\n", - "start_time: 2023-01-29 22:18:07.789282\n", - "end_time: 2023-01-29 22:18:07.914328\n", + "start_time: 2024-01-05 14:48:31.791928\n", + "end_time: 2024-01-05 14:48:32.501517\n", "\n", - "results_dir: /Users/mini-me/agnostiq/covalent/doc/source/how_to/coding/results\n", - "dispatch_id: 024088b9-6f74-4e5f-9757-9f088bd16b29\n", + "results_dir: /Users/sankalpsanand/.cache/covalent/results/bbd6f2b0-ecb4-4e05-ba63-6c10da81ceeb\n", + "dispatch_id: bbd6f2b0-ecb4-4e05-ba63-6c10da81ceeb\n", "\n", "Node Outputs\n", "------------\n", - "my_file_transfer_task(0): /Users/mini-me/agnostiq/covalent/doc/source/how_to/coding/my_dest_file\n", + "my_file_transfer_task(0): /Users/sankalpsanand/dev/covalent/doc/source/how_to/coding/my_dest_file\n", + ":postprocess:reconstruct(1): /Users/sankalpsanand/dev/covalent/doc/source/how_to/coding/my_dest_file\n", "\n", - "Reading from /Users/mini-me/agnostiq/covalent/doc/source/how_to/coding/my_dest_file \n", + "Reading from /Users/sankalpsanand/dev/covalent/doc/source/how_to/coding/my_dest_file \n", "\n", "Mares eat oats and does eat oats\n", "And little lambs eat ivy ...\n", @@ -265,7 +266,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/how_to/coding/file_transfers_for_workflows_to_remote.ipynb b/doc/source/how_to/coding/file_transfers_for_workflows_to_remote.ipynb deleted file mode 100644 index 1a44b623a..000000000 --- a/doc/source/how_to/coding/file_transfers_for_workflows_to_remote.ipynb +++ /dev/null @@ -1,159 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5120ef0f-8510-4ad7-a0ea-2e306aec0064", - "metadata": { - "tags": [] - }, - "source": [ - "## Transferring Remote Files After a Workflow\n", - "\n", - "Transfer a local file to a remote host's filesystem using Rsync via SSH.\n", - "\n", - "### Prerequisites\n", - "\n", - "1. Define the read (source) file path. \n", - "2. Create a source file to transfer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a20575c4", - "metadata": {}, - "outputs": [], - "source": [ - "import covalent as ct \n", - "\n", - "from pathlib import Path\n", - "\n", - "# define source & destination filepaths \n", - "source_filepath = Path('./my_source_file').resolve()\n", - "\n", - "# create an example file\n", - "source_filepath.touch()" - ] - }, - { - "cell_type": "markdown", - "id": "1616c355", - "metadata": {}, - "source": [ - "### Procedure\n", - "\n", - "Transfer a local file located in `source_filepath` to a remote host's filesystem located at `/home/ubuntu/my_dest_file` using Rsync via SSH.\n", - "\n", - "1. Define an `Rsync` strategy with the remote host and path:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "78667649", - "metadata": {}, - "outputs": [], - "source": [ - "strategy = ct.fs_strategies.Rsync(user='ubuntu', host='44.202.86.215', private_key_path='/path/to/private/key')" - ] - }, - { - "cell_type": "markdown", - "id": "1971b885", - "metadata": {}, - "source": [ - "2. Define an electron, passing a Covalent `TransferToRemote` object to the `files` keyword argument in the decorator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4abc0a90", - "metadata": {}, - "outputs": [], - "source": [ - "@ct.electron(\n", - " files=[ct.fs.TransferToRemote('/home/ubuntu/my_dest_file', str(source_filepath), strategy=strategy)]\n", - ")\n", - "def my_remote_file_transfer_task(files=[]):\n", - " pass\n" - ] - }, - { - "cell_type": "markdown", - "id": "2886440f", - "metadata": {}, - "source": [ - "3. Define a lattice in which to dispatch the workflow:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "85e52654", - "metadata": {}, - "outputs": [], - "source": [ - "@ct.lattice()\n", - "def my_workflow():\n", - " return my_remote_file_transfer_task()\n", - "\n", - "ct.dispatch(my_workflow)()" - ] - }, - { - "cell_type": "markdown", - "id": "57700440", - "metadata": {}, - "source": [ - "The file located at `source_filepath` is transferred to the specified file path (`/home/ubuntu/my_dest_file`) on host `44.202.86.215`. The transfer operation occurs after the workflow completes. In a typical real-world scenario, the transfer is used to move data generated by the workflow.\n" - ] - }, - { - "cell_type": "markdown", - "id": "244061a2", - "metadata": {}, - "source": [ - "### See Also\n", - "\n", - "[Transferring Local Files During Workflows](./file_transfers_for_workflows_local.ipynb)\n", - "\n", - "[Transferring Files to and from an S3 Bucket](/file_transfers_for_workflows_to_from_s3.ipynb)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "492834f4", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/doc/source/how_to/coding/file_transfers_to_from_azure_blob.ipynb b/doc/source/how_to/coding/file_transfers_to_from_azure_blob.ipynb index a0f85e837..3771a6e76 100644 --- a/doc/source/how_to/coding/file_transfers_to_from_azure_blob.ipynb +++ b/doc/source/how_to/coding/file_transfers_to_from_azure_blob.ipynb @@ -102,7 +102,7 @@ "\n", " # Save the grayscale image to the upload file path\n", " gray_image_path = files[1][0] # source filepath of second file transfer, to be uploaded\n", - " io.imsave(gray_image_path, gray_img)" + " io.imsave(gray_image_path, gray_img, mode=\"L\")" ] }, { diff --git a/doc/source/how_to/coding/file_transfers_to_from_gcp_storage.ipynb b/doc/source/how_to/coding/file_transfers_to_from_gcp_storage.ipynb index 5e1c02a22..83b08f2dc 100644 --- a/doc/source/how_to/coding/file_transfers_to_from_gcp_storage.ipynb +++ b/doc/source/how_to/coding/file_transfers_to_from_gcp_storage.ipynb @@ -52,8 +52,8 @@ "object_source_path = f\"gs://{storage_bucket}/remote_{unprocessed_filename}\"\n", "object_dest_path = f\"gs://{storage_bucket}/remote_{processed_filename}\"\n", "\n", - "ft_1 = ct.fs.TransferFromRemote(blob_source_path, unprocessed_filepath, strategy=strategy)\n", - "ft_2 = ct.fs.TransferToRemote(blob_dest_path, processed_filepath, strategy=strategy)" + "ft_1 = ct.fs.TransferFromRemote(object_source_path, unprocessed_filepath, strategy=strategy)\n", + "ft_2 = ct.fs.TransferToRemote(object_dest_path, processed_filepath, strategy=strategy)" ] }, { @@ -100,7 +100,7 @@ "\n", " # Save the grayscale image to the upload file path\n", " gray_image_path = files[1][0] # source filepath of second file transfer, to be uploaded\n", - " io.imsave(gray_image_path, gray_img)" + " io.imsave(gray_image_path, gray_img, mode=\"L\")" ] }, { @@ -116,7 +116,15 @@ "execution_count": 3, "id": "5831c062", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "COMPLETED\n" + ] + } + ], "source": [ "@ct.lattice\n", "def process_blob_data():\n", @@ -160,7 +168,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.8.18" } }, "nbformat": 4, diff --git a/doc/source/how_to/coding/file_transfers_to_from_remote.ipynb b/doc/source/how_to/coding/file_transfers_to_from_remote.ipynb index acfbe7040..d07489b3c 100644 --- a/doc/source/how_to/coding/file_transfers_to_from_remote.ipynb +++ b/doc/source/how_to/coding/file_transfers_to_from_remote.ipynb @@ -25,6 +25,8 @@ "\n", "### Procedure\n", "\n", + "Make sure to swap the `private_key` and `remote_host` values with your own.\n", + "\n", "1. Define an `Rsync` strategy with the remote host and private key path to be used for SSH." ] }, @@ -112,7 +114,7 @@ "\n", " # Save the grayscale image to the upload file path\n", " gray_image_path = files[1][0] # source filepath of second file transfer, to be uploaded\n", - " io.imsave(gray_image_path, gray_img)\n" + " io.imsave(gray_image_path, gray_img, mode=\"L\")\n" ] }, { @@ -191,7 +193,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/how_to/coding/file_transfers_to_from_s3.ipynb b/doc/source/how_to/coding/file_transfers_to_from_s3.ipynb index e50a2ca63..58710f911 100644 --- a/doc/source/how_to/coding/file_transfers_to_from_s3.ipynb +++ b/doc/source/how_to/coding/file_transfers_to_from_s3.ipynb @@ -94,7 +94,7 @@ "\n", " # Save the grayscale image to the upload file path\n", " gray_image_path = files[1][0] # source filepath of second file transfer, to be uploaded\n", - " io.imsave(gray_image_path, gray_img)\n" + " io.imsave(gray_image_path, gray_img, mode=\"L\")\n" ] }, { @@ -173,7 +173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/how_to/coding/file_transfers_to_remote.ipynb b/doc/source/how_to/coding/file_transfers_to_remote.ipynb deleted file mode 100644 index 7978bce6f..000000000 --- a/doc/source/how_to/coding/file_transfers_to_remote.ipynb +++ /dev/null @@ -1,182 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5120ef0f-8510-4ad7-a0ea-2e306aec0064", - "metadata": { - "tags": [] - }, - "source": [ - "## Transferring Remote Files After a Workflow\n", - "\n", - "Transfer a local file to a remote host's filesystem using Rsync via SSH.\n", - "\n", - "### Prerequisites\n", - "\n", - "1. Define the read (source) file path. \n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "2febd9b7", - "metadata": {}, - "outputs": [], - "source": [ - "import covalent as ct \n", - "\n", - "from pathlib import Path\n", - "\n", - "# define a source filepath\n", - "source_filepath = Path('./my_source_file').resolve()\n" - ] - }, - { - "cell_type": "markdown", - "id": "94db0ac7", - "metadata": {}, - "source": [ - "2. Create a source file to transfer." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "a20575c4", - "metadata": {}, - "outputs": [], - "source": [ - "# create an example file\n", - "source_filepath.touch()" - ] - }, - { - "cell_type": "markdown", - "id": "1616c355", - "metadata": {}, - "source": [ - "### Procedure\n", - "\n", - "Transfer a local file located in `source_filepath` to a remote host's filesystem located at `/home/ubuntu/my_dest_file` using Rsync via SSH.\n", - "\n", - "1. Define an `Rsync` strategy with the remote host and path:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "78667649", - "metadata": {}, - "outputs": [], - "source": [ - "strategy = ct.fs_strategies.Rsync(user='ubuntu', host='44.202.86.215', private_key_path='/path/to/private/key')" - ] - }, - { - "cell_type": "markdown", - "id": "1971b885", - "metadata": {}, - "source": [ - "2. Define an electron, passing a Covalent `TransferToRemote` object to the `files` keyword argument in the decorator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4abc0a90", - "metadata": {}, - "outputs": [], - "source": [ - "@ct.electron(\n", - " files=[ct.fs.TransferToRemote('/home/ubuntu/my_dest_file', str(source_filepath), strategy=strategy)]\n", - ")\n", - "def my_remote_file_transfer_task(files=[]):\n", - " pass\n" - ] - }, - { - "cell_type": "markdown", - "id": "2886440f", - "metadata": {}, - "source": [ - "3. Define a lattice in which to execute the workflow:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "85e52654", - "metadata": {}, - "outputs": [], - "source": [ - "@ct.lattice()\n", - "def my_workflow():\n", - " return my_remote_file_transfer_task()" - ] - }, - { - "cell_type": "markdown", - "id": "028c0fc4", - "metadata": {}, - "source": [ - "4. Dispatch the lattice:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b123c74", - "metadata": {}, - "outputs": [], - "source": [ - "ct.dispatch(my_workflow)()" - ] - }, - { - "cell_type": "markdown", - "id": "57700440", - "metadata": {}, - "source": [ - "The file located at `source_filepath` is transferred to the specified file path (`/home/ubuntu/my_dest_file`) on host `44.202.86.215`. The transfer operation occurs after the workflow completes. In a typical real-world scenario, the transfer is used to move data generated by the workflow.\n" - ] - }, - { - "cell_type": "markdown", - "id": "244061a2", - "metadata": {}, - "source": [ - "### See Also\n", - "\n", - "[Transferring Local Files During Workflows](./file_transfers_for_workflows_local.ipynb)\n", - "\n", - "[Transferring Files to and from an S3 Bucket](./file_transfers_to_from_s3.ipynb)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/doc/source/how_to/coding/processed_file.png b/doc/source/how_to/coding/processed_file.png index 72fcf858e..d8c5a7ab0 100644 Binary files a/doc/source/how_to/coding/processed_file.png and b/doc/source/how_to/coding/processed_file.png differ diff --git a/doc/source/how_to/execution/trigger_database.ipynb b/doc/source/how_to/execution/trigger_database.ipynb index bce8da792..629c77486 100644 --- a/doc/source/how_to/execution/trigger_database.ipynb +++ b/doc/source/how_to/execution/trigger_database.ipynb @@ -10,14 +10,11 @@ "\n", "## Prerequisites\n", " \n", - "1. Install the recommended SQL drivers that support SQLAlchemy. \n", - "2. Create an environment variable named `COVALENT_DATABASE_URL` and set the desired database file or URL. For the PostgreSQL instance, the database connection URL will be similar to the below code snippet,\n", - "```\n", - " export COVALENT_DATABASE_URL=postgresql+pg8000://:@:/\n", - "```\n", - "3. To migrate tables, use `covalent db migrate` to create the required tables in the mentioned database.\n", - "4. Then start covalent using `covalent start`. Now, the covalent server points to the new database.\n", - "5. Import the Covalent and the trigger." + "1. Install the recommended SQL drivers that support SQLAlchemy.\n", + "2. Make sure a user with the name `postgres` exists in your PostgreSQL database (you can run `createuser postgres` command for that).\n", + "3. Create a database called `aqdb` (you can run `createdb aqdb` command for this).\n", + "(both the commands mentioned above come installed as part of PostgreSQL installation as tested in macOS `postgres` installation using homebrew)\n", + "5. Import the Covalent and the DatabaseTrigger." ] }, { @@ -48,24 +45,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-10-04 08:34:30,374 INFO sqlalchemy.engine.Engine select pg_catalog.version()\n", - "2023-10-04 08:34:30,377 INFO sqlalchemy.engine.Engine [raw sql] ()\n", - "2023-10-04 08:34:30,385 INFO sqlalchemy.engine.Engine select current_schema()\n", - "2023-10-04 08:34:30,389 INFO sqlalchemy.engine.Engine [raw sql] ()\n", - "2023-10-04 08:34:30,391 INFO sqlalchemy.engine.Engine show standard_conforming_strings\n", - "2023-10-04 08:34:30,392 INFO sqlalchemy.engine.Engine [raw sql] ()\n", - "2023-10-04 08:34:30,397 INFO sqlalchemy.engine.Engine BEGIN (implicit)\n", - "2023-10-04 08:34:30,405 INFO sqlalchemy.engine.Engine select relname from pg_class c join pg_namespace n on n.oid=c.relnamespace where pg_catalog.pg_table_is_visible(c.oid) and relname=%s\n", - "2023-10-04 08:34:30,406 INFO sqlalchemy.engine.Engine [generated in 0.00120s] ('test_db_trigger',)\n", - "2023-10-04 08:34:30,410 INFO sqlalchemy.engine.Engine \n", + "2024-01-07 17:34:25,040 INFO sqlalchemy.engine.Engine select pg_catalog.version()\n", + "2024-01-07 17:34:25,041 INFO sqlalchemy.engine.Engine [raw sql] ()\n", + "2024-01-07 17:34:25,042 INFO sqlalchemy.engine.Engine select current_schema()\n", + "2024-01-07 17:34:25,042 INFO sqlalchemy.engine.Engine [raw sql] ()\n", + "2024-01-07 17:34:25,043 INFO sqlalchemy.engine.Engine show standard_conforming_strings\n", + "2024-01-07 17:34:25,043 INFO sqlalchemy.engine.Engine [raw sql] ()\n", + "2024-01-07 17:34:25,044 INFO sqlalchemy.engine.Engine BEGIN (implicit)\n", + "2024-01-07 17:34:25,044 INFO sqlalchemy.engine.Engine select relname from pg_class c join pg_namespace n on n.oid=c.relnamespace where pg_catalog.pg_table_is_visible(c.oid) and relname=%s\n", + "2024-01-07 17:34:25,045 INFO sqlalchemy.engine.Engine [generated in 0.00025s] ('test_db_trigger',)\n", + "2024-01-07 17:34:25,046 INFO sqlalchemy.engine.Engine \n", "CREATE TABLE test_db_trigger (\n", "\ttrigger_at TIMESTAMP WITHOUT TIME ZONE NOT NULL, \n", "\tPRIMARY KEY (trigger_at)\n", ")\n", "\n", "\n", - "2023-10-04 08:34:30,411 INFO sqlalchemy.engine.Engine [no key 0.00107s] ()\n", - "2023-10-04 08:34:30,448 INFO sqlalchemy.engine.Engine COMMIT\n" + "2024-01-07 17:34:25,046 INFO sqlalchemy.engine.Engine [no key 0.00021s] ()\n", + "2024-01-07 17:34:25,049 INFO sqlalchemy.engine.Engine COMMIT\n" ] } ], @@ -102,17 +99,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-10-04 08:34:33,407 INFO sqlalchemy.engine.Engine INSERT INTO test_db_trigger (trigger_at) VALUES (%s)\n", - "2023-10-04 08:34:33,409 INFO sqlalchemy.engine.Engine [generated in 0.00208s] ((datetime.datetime(2023, 10, 4, 14, 4, 33, 406624),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406636),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406638),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406639),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406641),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406643),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406645),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406646),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406648),), (datetime.datetime(2023, 10, 4, 14, 4, 33, 406650),))\n", - "2023-10-04 08:34:33,423 INFO sqlalchemy.engine.Engine COMMIT\n" + "2024-01-07 17:34:28,071 INFO sqlalchemy.engine.Engine INSERT INTO test_db_trigger (trigger_at) VALUES (%s)\n", + "2024-01-07 17:34:28,071 INFO sqlalchemy.engine.Engine [generated in 0.00094s] ((datetime.datetime(2024, 1, 7, 12, 34, 27, 33553),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 138484),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 243426),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 348190),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 453000),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 555778),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 655854),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 760891),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 862814),), (datetime.datetime(2024, 1, 7, 12, 34, 27, 965930),))\n", + "2024-01-07 17:34:28,079 INFO sqlalchemy.engine.Engine COMMIT\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/tmp/ipykernel_146652/3174457459.py:6: RemovedIn20Warning: Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0. To prevent incompatible upgrades prior to updating applications, ensure requirements files are pinned to \"sqlalchemy<2.0\". Set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. Set environment variable SQLALCHEMY_SILENCE_UBER_WARNING=1 to silence this message. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9)\n", - " result = conn.execute(insert(test_db_trigger),[*values])#{\"trigger_at\": trigger_at}\n" + "/var/folders/1z/64_91wwj46ng1xjffddgz_n40000gn/T/ipykernel_2625/3058673604.py:11: RemovedIn20Warning: Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0. To prevent incompatible upgrades prior to updating applications, ensure requirements files are pinned to \"sqlalchemy<2.0\". Set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. Set environment variable SQLALCHEMY_SILENCE_UBER_WARNING=1 to silence this message. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9)\n", + " result = conn.execute(insert(test_db_trigger),[*values])\n" ] } ], @@ -120,8 +117,13 @@ "# load sample data.\n", "from sqlalchemy import insert\n", "from datetime import datetime\n", + "import time\n", + "\n", "with engine.connect() as conn:\n", - " values = [{\"trigger_at\": datetime.now()} for _ in range(10)]\n", + " values = []\n", + " for _ in range(10):\n", + " values.append({\"trigger_at\": datetime.now()})\n", + " time.sleep(0.1)\n", " result = conn.execute(insert(test_db_trigger),[*values])" ] }, @@ -179,7 +181,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "de35492d-4f51-473e-b814-ad203939f85a\n" + "a7a74b5e-e865-430e-9a6c-258e8c51d0ed\n" ] } ], @@ -227,21 +229,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "[2023-10-04 08:36:10,622] [DEBUG] local.py: Line 334 in stop_triggers: Triggers for following dispatch_ids have stopped observing:\n", - "[2023-10-04 08:36:10,627] [DEBUG] local.py: Line 336 in stop_triggers: de35492d-4f51-473e-b814-ad203939f85a\n", - "2023-10-04 08:36:10,630 INFO sqlalchemy.engine.Engine BEGIN (implicit)\n", - "2023-10-04 08:36:10,633 INFO sqlalchemy.engine.Engine select relname from pg_class c join pg_namespace n on n.oid=c.relnamespace where pg_catalog.pg_table_is_visible(c.oid) and relname=%s\n", - "2023-10-04 08:36:10,637 INFO sqlalchemy.engine.Engine [cached since 100.2s ago] ('test_db_trigger',)\n", - "2023-10-04 08:36:10,643 INFO sqlalchemy.engine.Engine \n", + "2024-01-07 17:35:51,633 INFO sqlalchemy.engine.Engine BEGIN (implicit)\n", + "2024-01-07 17:35:51,633 INFO sqlalchemy.engine.Engine select relname from pg_class c join pg_namespace n on n.oid=c.relnamespace where pg_catalog.pg_table_is_visible(c.oid) and relname=%s\n", + "2024-01-07 17:35:51,634 INFO sqlalchemy.engine.Engine [cached since 86.59s ago] ('test_db_trigger',)\n", + "2024-01-07 17:35:51,635 INFO sqlalchemy.engine.Engine \n", "DROP TABLE test_db_trigger\n", - "2023-10-04 08:36:10,645 INFO sqlalchemy.engine.Engine [no key 0.00186s] ()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-10-04 08:36:11,073 INFO sqlalchemy.engine.Engine COMMIT\n" + "2024-01-07 17:35:51,635 INFO sqlalchemy.engine.Engine [no key 0.00026s] ()\n", + "2024-01-07 17:35:51,698 INFO sqlalchemy.engine.Engine COMMIT\n" ] } ], @@ -271,13 +265,6 @@ "source": [ "###### " ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -296,7 +283,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.17" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/how_to/execution/trigger_dir.ipynb b/doc/source/how_to/execution/trigger_dir.ipynb index ae764a907..f01af9b54 100644 --- a/doc/source/how_to/execution/trigger_dir.ipynb +++ b/doc/source/how_to/execution/trigger_dir.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -88,14 +88,14 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "656213bc-ebd3-444b-9c48-1542c497061a\n" + "6e3a3e60-f105-48b9-95ad-fdff968a3f2f\n" ] } ], @@ -143,13 +143,6 @@ "\n", "[Adding a SQLite Trigger to a Lattice](./trigger_sqlite.ipynb)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -168,7 +161,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/assets/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/assets/requirements.txt index 67e90aa15..872846975 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/assets/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/assets/requirements.txt @@ -1,5 +1,4 @@ covalent matplotlib==3.6.3 -pennylane==0.25.1 scikit-learn==1.0.2 torch==1.13.1 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/source-jvsc-b1861ceb-413c-44fd-9049-76f027265d183c7f2d69-2f7c-43b9-9202-d5b51c50bb22.ipynb b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/source-jvsc-b1861ceb-413c-44fd-9049-76f027265d183c7f2d69-2f7c-43b9-9202-d5b51c50bb22.ipynb deleted file mode 100644 index 363fcab7e..000000000 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_ensemble_classification/source-jvsc-b1861ceb-413c-44fd-9049-76f027265d183c7f2d69-2f7c-43b9-9202-d5b51c50bb22.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_hybrid/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_hybrid/requirements.txt index b789655ae..a2af84b92 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_hybrid/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_hybrid/requirements.txt @@ -1,4 +1,3 @@ covalent matplotlib==3.4.3 -pennylane==0.25.1 pennylane-sf==0.20.1 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_iris_classification/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_iris_classification/requirements.txt index b789655ae..a2af84b92 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_iris_classification/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_iris_classification/requirements.txt @@ -1,4 +1,3 @@ covalent matplotlib==3.4.3 -pennylane==0.25.1 pennylane-sf==0.20.1 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_kernel/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_kernel/requirements.txt index 2d9f9b9bc..c167e5c1a 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_kernel/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_kernel/requirements.txt @@ -1,7 +1,6 @@ covalent matplotlib==3.5.1 numpy==1.24.2 -pennylane==0.25.1 pennylane-sf==0.20.1 scikit-learn==1.0.2 torch==2.0.0 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_parity_classifier/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_parity_classifier/requirements.txt index 1b91aa61e..8eb2fbc39 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/pennylane_parity_classifier/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/pennylane_parity_classifier/requirements.txt @@ -1,3 +1,2 @@ covalent matplotlib==3.4.3 -pennylane==0.25.1 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/qaoa_maxcut/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/qaoa_maxcut/requirements.txt index 7222681ea..8eb2fbc39 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/qaoa_maxcut/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/qaoa_maxcut/requirements.txt @@ -1,3 +1,2 @@ covalent matplotlib==3.4.3 -pennylane==0.26.0 diff --git a/doc/source/tutorials/1_QuantumMachineLearning/quantum_embedding_kernel/requirements.txt b/doc/source/tutorials/1_QuantumMachineLearning/quantum_embedding_kernel/requirements.txt index 1eba2fa34..f11dad26f 100644 --- a/doc/source/tutorials/1_QuantumMachineLearning/quantum_embedding_kernel/requirements.txt +++ b/doc/source/tutorials/1_QuantumMachineLearning/quantum_embedding_kernel/requirements.txt @@ -1,4 +1,3 @@ covalent matplotlib==3.5.1 -pennylane==0.23.1 scikit-learn==1.1.1 diff --git a/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/requirements.txt b/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/requirements.txt index dafc1665c..6a986e5b3 100644 --- a/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/requirements.txt +++ b/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/requirements.txt @@ -1,2 +1,2 @@ covalent -pymatgen[relaxation]==2023.5.10 +pymatgen[relaxation]==2023.9.25 diff --git a/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/source.ipynb b/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/source.ipynb index 34af156e9..71ab685eb 100644 --- a/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/source.ipynb +++ b/doc/source/tutorials/3_QuantumChemistry/dynamic_quantum_chemistry/source.ipynb @@ -7,6 +7,8 @@ "source": [ "# Dynamic Quantum Chemistry Workflow \n", "\n", + "NOTE: Make sure to use Python 3.9 or higher for this, `pymatgen` package does not support Python 3.8 or lower.\n", + "\n", "In this tutorial, we discuss how _covalent_ can be used to construct and manage dynamic workflows that are common in quantum-chemical simulations. In this example, given a crystal structure, we seek to do the following:\n", "\n", "1. Relax the atomic positions and cell volume for the structure (i.e. find the local minimum energy configuration).\n", @@ -34,7 +36,7 @@ "output_type": "stream", "text": [ "covalent\n", - "pymatgen[relaxation]==2023.5.10\n" + "pymatgen[relaxation]==2023.9.25\n" ] } ], @@ -44,6 +46,17 @@ " print(line.rstrip())" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you're using mac then you might also have to install the following packages with homebrew as:\n", + "\n", + "```\n", + "brew install freetype pkg-config libpng\n", + "``````" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -60,7 +73,6 @@ "source": [ "import covalent as ct\n", "\n", - "from subprocess import Popen\n", "from pymatgen.core import Structure\n", "from pymatgen.core.surface import generate_all_slabs" ] @@ -82,12 +94,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Covalent server has started at http://localhost:48008\n" + "Covalent server is already running.\n" ] } ], "source": [ - "sts = Popen(\"covalent start\", shell=True).wait()" + "ct.covalent_start()" ] }, { @@ -168,7 +180,7 @@ "metadata": {}, "outputs": [], "source": [ - "@ct.lattice(executor=\"local\")\n", + "@ct.lattice\n", "def workflow(structure):\n", " relaxed_structure = relax_structure(structure)\n", " slabs = carve_slabs(relaxed_structure)\n", @@ -258,96 +270,96 @@ "status: COMPLETED\n", "result: [Structure Summary\n", "Lattice\n", - " abc : 3.012274887854692 3.0122748878546917 30.122748878546922\n", - " angles : 120.00000000000001 120.00000000000001 60.00000000000001\n", - " volume : 193.27193999999986\n", - " A : 2.6087065760640837 0.0 -1.5061374439273467\n", - " B : 0.8695688586880268 2.459512146747805 -1.5061374439273467\n", + " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", + " angles : 120.00000000000001 120.00000000000001 59.99999999999999\n", + " volume : 193.27193999999994\n", + " A : 2.608706576064084 0.0 -1.5061374439273467\n", + " B : 0.8695688586880276 2.4595121467478056 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0808, -0.0571, 2.1193) [-0.0232, -0.0232, 0.0680]\n", - "PeriodicSite: O (1.8010, 1.2735, -0.6459) [0.5178, 0.5178, 0.0303]\n", - "PeriodicSite: Mg (-0.0401, -0.0284, 5.2020) [-0.0115, -0.0115, 0.1715]\n", - "PeriodicSite: O (1.7790, 1.2579, 2.3282) [0.5114, 0.5114, 0.1284]\n", - "PeriodicSite: Mg (-0.0392, -0.0277, 8.2159) [-0.0113, -0.0113, 0.2716]\n", - "PeriodicSite: O (1.7778, 1.2571, 5.3384) [0.5111, 0.5111, 0.2283]\n", - "PeriodicSite: Mg (-0.0320, -0.0226, 11.2406) [-0.0092, -0.0092, 0.3722]\n", - "PeriodicSite: O (1.7829, 1.2607, 8.3596) [0.5126, 0.5126, 0.3288]\n", - "PeriodicSite: Mg (-0.0059, -0.0042, 14.2980) [-0.0017, -0.0017, 0.4745]\n", - "PeriodicSite: O (1.7529, 1.2395, 11.3199) [0.5040, 0.5040, 0.4262], Structure Summary\n", + "PeriodicSite: Mg (-0.06335, -0.04479, 2.149) [-0.01821, -0.01821, 0.06954]\n", + "PeriodicSite: O (1.804, 1.275, -0.6412) [0.5186, 0.5186, 0.03057]\n", + "PeriodicSite: Mg (-0.04308, -0.03047, 5.197) [-0.01239, -0.01239, 0.1713]\n", + "PeriodicSite: O (1.762, 1.246, 2.298) [0.5064, 0.5064, 0.1269]\n", + "PeriodicSite: Mg (-0.03833, -0.0271, 8.217) [-0.01102, -0.01102, 0.2717]\n", + "PeriodicSite: O (1.776, 1.255, 5.335) [0.5105, 0.5105, 0.2281]\n", + "PeriodicSite: Mg (-0.03796, -0.02684, 11.23) [-0.01091, -0.01091, 0.3717]\n", + "PeriodicSite: O (1.788, 1.265, 8.369) [0.5141, 0.5141, 0.3292]\n", + "PeriodicSite: Mg (-0.01275, -0.009017, 14.29) [-0.003666, -0.003666, 0.4739]\n", + "PeriodicSite: O (1.762, 1.246, 11.34) [0.5066, 0.5066, 0.427], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", - " angles : 120.00000000000001 60.00000000000001 90.0\n", + " angles : 120.00000000000001 59.99999999999999 90.0\n", " volume : 193.2719399999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", - " B : 0.8695688586880286 2.4595121467478047 -1.5061374439273467\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", + " B : 0.8695688586880287 2.459512146747805 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0015, 0.0021, 1.5088) [-0.0009, 0.0009, 0.0502]\n", - "PeriodicSite: O (1.7403, 1.2281, 1.5042) [0.5007, 0.4993, 0.0499]\n", - "PeriodicSite: Mg (0.0016, -0.0022, 4.5157) [0.0009, -0.0009, 0.1498]\n", - "PeriodicSite: O (1.7373, 1.2323, 4.5215) [0.4990, 0.5010, 0.1502]\n", - "PeriodicSite: Mg (0.0000, -0.0000, 7.5307) [0.0000, -0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 7.5307) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (-0.0016, 0.0022, 10.5457) [-0.0009, 0.0009, 0.3502]\n", - "PeriodicSite: O (1.7409, 1.2272, 10.5399) [0.5010, 0.4990, 0.3498]\n", - "PeriodicSite: Mg (0.0015, -0.0021, 13.5526) [0.0009, -0.0009, 0.4498]\n", - "PeriodicSite: O (1.7380, 1.2314, 13.5572) [0.4993, 0.5007, 0.4501], Structure Summary\n", + "PeriodicSite: Mg (-0.003667, 0.005186, 1.512) [-0.002108, 0.002108, 0.05042]\n", + "PeriodicSite: O (1.742, 1.226, 1.502) [0.5015, 0.4985, 0.0497]\n", + "PeriodicSite: Mg (0.004204, -0.005944, 4.511) [0.002417, -0.002417, 0.1495]\n", + "PeriodicSite: O (1.737, 1.233, 4.523) [0.4985, 0.5015, 0.1503]\n", + "PeriodicSite: Mg (-2.373e-07, 1.452e-07, 7.531) [-1.106e-07, 5.904e-08, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 7.531) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004206, 0.005949, 10.55) [-0.002419, 0.002419, 0.3505]\n", + "PeriodicSite: O (1.742, 1.226, 10.54) [0.5015, 0.4985, 0.3497]\n", + "PeriodicSite: Mg (0.003665, -0.005183, 13.55) [0.002107, -0.002107, 0.4496]\n", + "PeriodicSite: O (1.737, 1.233, 13.56) [0.4985, 0.5015, 0.4503], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 48.196398205675074\n", - " angles : 30.000000000000014 60.00000000000002 73.22134511903965\n", + " angles : 30.000000000000014 59.99999999999999 73.22134511903964\n", " volume : 309.2351039999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : -0.869568858688028 2.4595121467478056 4.518412331782038\n", " C : 0.0 0.0 48.196398205675074\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0262, 0.0105, 2.2139) [0.0114, 0.0042, 0.0452]\n", - "PeriodicSite: O (0.9825, 1.0539, 3.5697) [0.5195, 0.4285, 0.0177]\n", - "PeriodicSite: Mg (0.0662, 0.2336, 5.1568) [0.0571, 0.0950, 0.0963]\n", - "PeriodicSite: O (0.8459, 1.1867, 6.8186) [0.4851, 0.4825, 0.0811]\n", - "PeriodicSite: Mg (0.0133, 0.0507, 8.2607) [0.0120, 0.0206, 0.1691]\n", - "PeriodicSite: O (0.8496, 1.1727, 9.8246) [0.4846, 0.4768, 0.1440]\n", - "PeriodicSite: Mg (0.0086, 0.0959, 11.2811) [0.0163, 0.0390, 0.2299]\n", - "PeriodicSite: O (0.8389, 1.1668, 12.8553) [0.4797, 0.4744, 0.2073]\n", - "PeriodicSite: Mg (0.0091, 0.0613, 14.2926) [0.0118, 0.0249, 0.2938]\n", - "PeriodicSite: O (0.8384, 1.1314, 15.8685) [0.4747, 0.4600, 0.2713]\n", - "PeriodicSite: Mg (0.0019, 0.0338, 17.3173) [0.0053, 0.0137, 0.3579]\n", - "PeriodicSite: O (0.8477, 1.1616, 18.8647) [0.4824, 0.4723, 0.3321]\n", - "PeriodicSite: Mg (0.0237, 0.0330, 20.2919) [0.0135, 0.0134, 0.4193]\n", - "PeriodicSite: O (0.8368, 1.0796, 21.8957) [0.4671, 0.4390, 0.3986]\n", - "PeriodicSite: Mg (-0.0806, 0.1485, 23.4847) [-0.0108, 0.0604, 0.4819]\n", - "PeriodicSite: O (0.8484, 1.2181, 24.8879) [0.4903, 0.4953, 0.4546], Structure Summary\n", + "PeriodicSite: Mg (0.1339, -0.01997, 2.027) [0.04861, -0.008121, 0.04131]\n", + "PeriodicSite: O (1.084, 1.0, 3.395) [0.5509, 0.4068, 0.01508]\n", + "PeriodicSite: Mg (0.1856, 0.2225, 4.95) [0.1013, 0.09047, 0.09106]\n", + "PeriodicSite: O (0.9559, 1.166, 6.628) [0.5245, 0.4742, 0.07668]\n", + "PeriodicSite: Mg (0.09665, 0.08853, 8.116) [0.04905, 0.036, 0.1635]\n", + "PeriodicSite: O (0.9216, 1.175, 9.7) [0.5125, 0.4777, 0.1405]\n", + "PeriodicSite: Mg (0.04673, 0.1156, 11.22) [0.03359, 0.04702, 0.2272]\n", + "PeriodicSite: O (0.8436, 1.125, 12.85) [0.4758, 0.4574, 0.2088]\n", + "PeriodicSite: Mg (-0.005373, 0.07126, 14.32) [0.007599, 0.02897, 0.2941]\n", + "PeriodicSite: O (0.7974, 1.125, 15.94) [0.4581, 0.4574, 0.2735]\n", + "PeriodicSite: Mg (-0.06382, 0.06227, 17.43) [-0.01603, 0.02532, 0.3598]\n", + "PeriodicSite: O (0.7609, 1.136, 19.01) [0.4456, 0.4618, 0.3373]\n", + "PeriodicSite: Mg (-0.07782, 0.05423, 20.47) [-0.02248, 0.02205, 0.4233]\n", + "PeriodicSite: O (0.7283, 1.092, 22.08) [0.4271, 0.4439, 0.4032]\n", + "PeriodicSite: Mg (-0.1775, 0.196, 23.65) [-0.04147, 0.07969, 0.4846]\n", + "PeriodicSite: O (0.727, 1.229, 25.1) [0.4452, 0.4996, 0.46], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 42.17184842996569\n", - " angles : 29.999999999999964 59.999999999999986 54.735610317245346\n", + " angles : 29.99999999999997 59.999999999999986 54.735610317245346\n", " volume : 270.58071599999977\n", " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : 0.8695688586880266 2.4595121467478043 4.51841233178204\n", " C : 0.0 0.0 42.17184842996569\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0254, -0.0718, 1.5501) [0.0000, -0.0292, 0.0399]\n", - "PeriodicSite: O (1.7306, 1.2057, 4.5332) [0.5000, 0.4902, 0.0371]\n", - "PeriodicSite: Mg (0.0279, 0.0790, 4.4700) [0.0000, 0.0321, 0.1026]\n", - "PeriodicSite: O (1.7433, 1.2415, 7.5235) [0.5000, 0.5048, 0.1065]\n", - "PeriodicSite: Mg (-0.0041, -0.0115, 7.5377) [0.0000, -0.0047, 0.1792]\n", - "PeriodicSite: O (1.7393, 1.2301, 10.5427) [0.5000, 0.5002, 0.1785]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 10.5430) [0.0000, 0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 13.5552) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (0.0041, 0.0115, 13.5482) [0.0000, 0.0047, 0.3208]\n", - "PeriodicSite: O (1.7390, 1.2294, 16.5678) [0.5000, 0.4998, 0.3215]\n", - "PeriodicSite: Mg (-0.0279, -0.0790, 16.6159) [0.0000, -0.0321, 0.3974]\n", - "PeriodicSite: O (1.7350, 1.2180, 19.5870) [0.5000, 0.4952, 0.3935]\n", - "PeriodicSite: Mg (0.0254, 0.0718, 19.5358) [-0.0000, 0.0292, 0.4601]\n", - "PeriodicSite: O (1.7477, 1.2539, 22.5773) [0.5000, 0.5098, 0.4629]]\n", - "input args: ['Full Formula (Mg1 O1)\\nReduced Formula: MgO\\nabc : 3.012275 3.012275 3.012275\\nangles: 60.000000 60.000000 60.000000\\npbc : True True True\\nSites (2)\\n # SP a b c\\n--- ---- --- --- ---\\n 0 Mg 0 0 0\\n 1 O 0.5 0.5 0.5']\n", - "input kwargs: {}\n", + "PeriodicSite: Mg (-0.02746, -0.07768, 1.554) [2.276e-07, -0.03158, 0.04023]\n", + "PeriodicSite: O (1.734, 1.216, 4.527) [0.5, 0.4945, 0.0365]\n", + "PeriodicSite: Mg (0.02554, 0.07223, 4.474) [1.088e-06, 0.02937, 0.1029]\n", + "PeriodicSite: O (1.746, 1.25, 7.518) [0.5, 0.5081, 0.106]\n", + "PeriodicSite: Mg (0.004413, 0.01248, 7.523) [1.247e-07, 0.005074, 0.1778]\n", + "PeriodicSite: O (1.736, 1.22, 10.55) [0.5, 0.4962, 0.1791]\n", + "PeriodicSite: Mg (5.245e-07, 2.488e-06, 10.54) [-1.362e-07, 1.012e-06, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 13.56) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004412, -0.01248, 13.56) [-5.311e-08, -0.005073, 0.3222]\n", + "PeriodicSite: O (1.742, 1.239, 16.56) [0.5, 0.5038, 0.3209]\n", + "PeriodicSite: Mg (-0.02554, -0.07224, 16.61) [-3.741e-08, -0.02937, 0.3971]\n", + "PeriodicSite: O (1.732, 1.21, 19.59) [0.5, 0.4919, 0.394]\n", + "PeriodicSite: Mg (0.02746, 0.07767, 19.53) [-2.13e-08, 0.03158, 0.4598]\n", + "PeriodicSite: O (1.744, 1.243, 22.58) [0.5, 0.5055, 0.4635]]\n", + "input args: None\n", + "input kwargs: None\n", "error: None\n", "\n", - "start_time: 2023-05-15 22:45:11.634086\n", - "end_time: 2023-05-15 22:46:21.630720\n", + "start_time: 2024-01-07 18:47:40.318085\n", + "end_time: 2024-01-07 18:47:50.077092\n", "\n", - "results_dir: /home/rosen/.local/share/covalent/data\n", - "dispatch_id: 190be75f-a9b0-431d-a7de-fea489b3b5d2\n", + "results_dir: /Users/neptune/.cache/covalent/results/0de284ad-fecb-4c76-ba03-c4dd8d8b3103\n", + "dispatch_id: 0de284ad-fecb-4c76-ba03-c4dd8d8b3103\n", "\n", "Node Outputs\n", "------------\n", @@ -382,255 +394,255 @@ " 1 O 0.5 0.5 0.5\n", "carve_slabs(2): [Structure Summary\n", "Lattice\n", - " abc : 3.012274887854692 3.0122748878546917 30.122748878546922\n", - " angles : 120.00000000000001 120.00000000000001 60.00000000000001\n", - " volume : 193.27193999999986\n", - " A : 2.6087065760640837 0.0 -1.5061374439273467\n", - " B : 0.8695688586880268 2.459512146747805 -1.5061374439273467\n", + " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", + " angles : 120.00000000000001 120.00000000000001 59.99999999999999\n", + " volume : 193.27193999999994\n", + " A : 2.608706576064084 0.0 -1.5061374439273467\n", + " B : 0.8695688586880276 2.4595121467478056 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0000, 0.0000, 2.2592) [0.0000, 0.0000, 0.0750]\n", - "PeriodicSite: O (1.7391, 1.2298, -0.7531) [0.5000, 0.5000, 0.0250]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 5.2715) [0.0000, 0.0000, 0.1750]\n", - "PeriodicSite: O (1.7391, 1.2298, 2.2592) [0.5000, 0.5000, 0.1250]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 8.2838) [0.0000, 0.0000, 0.2750]\n", - "PeriodicSite: O (1.7391, 1.2298, 5.2715) [0.5000, 0.5000, 0.2250]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 11.2960) [0.0000, 0.0000, 0.3750]\n", - "PeriodicSite: O (1.7391, 1.2298, 8.2838) [0.5000, 0.5000, 0.3250]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 14.3083) [0.0000, 0.0000, 0.4750]\n", - "PeriodicSite: O (1.7391, 1.2298, 11.2960) [0.5000, 0.5000, 0.4250], Structure Summary\n", + "PeriodicSite: Mg (0.0, 0.0, 2.259) [0.0, 0.0, 0.075]\n", + "PeriodicSite: O (1.739, 1.23, -0.7531) [0.5, 0.5, 0.025]\n", + "PeriodicSite: Mg (0.0, 0.0, 5.271) [0.0, 0.0, 0.175]\n", + "PeriodicSite: O (1.739, 1.23, 2.259) [0.5, 0.5, 0.125]\n", + "PeriodicSite: Mg (0.0, 0.0, 8.284) [0.0, 0.0, 0.275]\n", + "PeriodicSite: O (1.739, 1.23, 5.271) [0.5, 0.5, 0.225]\n", + "PeriodicSite: Mg (0.0, 0.0, 11.3) [0.0, 0.0, 0.375]\n", + "PeriodicSite: O (1.739, 1.23, 8.284) [0.5, 0.5, 0.325]\n", + "PeriodicSite: Mg (0.0, 0.0, 14.31) [0.0, 0.0, 0.475]\n", + "PeriodicSite: O (1.739, 1.23, 11.3) [0.5, 0.5, 0.425], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", - " angles : 120.00000000000001 60.00000000000001 90.0\n", + " angles : 120.00000000000001 59.99999999999999 90.0\n", " volume : 193.2719399999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", - " B : 0.8695688586880286 2.4595121467478047 -1.5061374439273467\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", + " B : 0.8695688586880287 2.459512146747805 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0000, 0.0000, 1.5061) [0.0000, 0.0000, 0.0500]\n", - "PeriodicSite: O (1.7391, 1.2298, 1.5061) [0.5000, 0.5000, 0.0500]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 4.5184) [0.0000, 0.0000, 0.1500]\n", - "PeriodicSite: O (1.7391, 1.2298, 4.5184) [0.5000, 0.5000, 0.1500]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 7.5307) [0.0000, 0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 7.5307) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 10.5430) [0.0000, 0.0000, 0.3500]\n", - "PeriodicSite: O (1.7391, 1.2298, 10.5430) [0.5000, 0.5000, 0.3500]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 13.5552) [0.0000, 0.0000, 0.4500]\n", - "PeriodicSite: O (1.7391, 1.2298, 13.5552) [0.5000, 0.5000, 0.4500], Structure Summary\n", + "PeriodicSite: Mg (0.0, 0.0, 1.506) [0.0, 0.0, 0.05]\n", + "PeriodicSite: O (1.739, 1.23, 1.506) [0.5, 0.5, 0.05]\n", + "PeriodicSite: Mg (0.0, 0.0, 4.518) [0.0, 0.0, 0.15]\n", + "PeriodicSite: O (1.739, 1.23, 4.518) [0.5, 0.5, 0.15]\n", + "PeriodicSite: Mg (0.0, 0.0, 7.531) [0.0, 0.0, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 7.531) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (0.0, 0.0, 10.54) [0.0, 0.0, 0.35]\n", + "PeriodicSite: O (1.739, 1.23, 10.54) [0.5, 0.5, 0.35]\n", + "PeriodicSite: Mg (0.0, 0.0, 13.56) [0.0, 0.0, 0.45]\n", + "PeriodicSite: O (1.739, 1.23, 13.56) [0.5, 0.5, 0.45], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 48.196398205675074\n", - " angles : 30.000000000000014 60.00000000000002 73.22134511903965\n", + " angles : 30.000000000000014 59.99999999999999 73.22134511903964\n", " volume : 309.2351039999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : -0.869568858688028 2.4595121467478056 4.518412331782038\n", " C : 0.0 0.0 48.196398205675074\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0000, 0.0000, 2.2592) [0.0000, 0.0000, 0.0469]\n", - "PeriodicSite: O (0.8696, 1.2298, 3.7653) [0.5000, 0.5000, 0.0156]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 5.2715) [0.0000, 0.0000, 0.1094]\n", - "PeriodicSite: O (0.8696, 1.2298, 6.7776) [0.5000, 0.5000, 0.0781]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 8.2838) [0.0000, 0.0000, 0.1719]\n", - "PeriodicSite: O (0.8696, 1.2298, 9.7899) [0.5000, 0.5000, 0.1406]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 11.2960) [0.0000, 0.0000, 0.2344]\n", - "PeriodicSite: O (0.8696, 1.2298, 12.8022) [0.5000, 0.5000, 0.2031]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 14.3083) [0.0000, 0.0000, 0.2969]\n", - "PeriodicSite: O (0.8696, 1.2298, 15.8144) [0.5000, 0.5000, 0.2656]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 17.3206) [0.0000, 0.0000, 0.3594]\n", - "PeriodicSite: O (0.8696, 1.2298, 18.8267) [0.5000, 0.5000, 0.3281]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 20.3329) [0.0000, 0.0000, 0.4219]\n", - "PeriodicSite: O (0.8696, 1.2298, 21.8390) [0.5000, 0.5000, 0.3906]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 23.3451) [0.0000, 0.0000, 0.4844]\n", - "PeriodicSite: O (0.8696, 1.2298, 24.8513) [0.5000, 0.5000, 0.4531], Structure Summary\n", + "PeriodicSite: Mg (0.0, 0.0, 2.259) [0.0, 0.0, 0.04688]\n", + "PeriodicSite: O (0.8696, 1.23, 3.765) [0.5, 0.5, 0.01562]\n", + "PeriodicSite: Mg (0.0, 0.0, 5.271) [0.0, 0.0, 0.1094]\n", + "PeriodicSite: O (0.8696, 1.23, 6.778) [0.5, 0.5, 0.07812]\n", + "PeriodicSite: Mg (0.0, 0.0, 8.284) [0.0, 0.0, 0.1719]\n", + "PeriodicSite: O (0.8696, 1.23, 9.79) [0.5, 0.5, 0.1406]\n", + "PeriodicSite: Mg (0.0, 0.0, 11.3) [0.0, 0.0, 0.2344]\n", + "PeriodicSite: O (0.8696, 1.23, 12.8) [0.5, 0.5, 0.2031]\n", + "PeriodicSite: Mg (0.0, 0.0, 14.31) [0.0, 0.0, 0.2969]\n", + "PeriodicSite: O (0.8696, 1.23, 15.81) [0.5, 0.5, 0.2656]\n", + "PeriodicSite: Mg (0.0, 0.0, 17.32) [0.0, 0.0, 0.3594]\n", + "PeriodicSite: O (0.8696, 1.23, 18.83) [0.5, 0.5, 0.3281]\n", + "PeriodicSite: Mg (0.0, 0.0, 20.33) [0.0, 0.0, 0.4219]\n", + "PeriodicSite: O (0.8696, 1.23, 21.84) [0.5, 0.5, 0.3906]\n", + "PeriodicSite: Mg (0.0, 0.0, 23.35) [0.0, 0.0, 0.4844]\n", + "PeriodicSite: O (0.8696, 1.23, 24.85) [0.5, 0.5, 0.4531], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 42.17184842996569\n", - " angles : 29.999999999999964 59.999999999999986 54.735610317245346\n", + " angles : 29.99999999999997 59.999999999999986 54.735610317245346\n", " volume : 270.58071599999977\n", " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : 0.8695688586880266 2.4595121467478043 4.51841233178204\n", " C : 0.0 0.0 42.17184842996569\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0000, 0.0000, 1.5061) [0.0000, 0.0000, 0.0357]\n", - "PeriodicSite: O (1.7391, 1.2298, 4.5184) [0.5000, 0.5000, 0.0357]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 4.5184) [0.0000, 0.0000, 0.1071]\n", - "PeriodicSite: O (1.7391, 1.2298, 7.5307) [0.5000, 0.5000, 0.1071]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 7.5307) [0.0000, 0.0000, 0.1786]\n", - "PeriodicSite: O (1.7391, 1.2298, 10.5430) [0.5000, 0.5000, 0.1786]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 10.5430) [0.0000, 0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 13.5552) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 13.5552) [0.0000, 0.0000, 0.3214]\n", - "PeriodicSite: O (1.7391, 1.2298, 16.5675) [0.5000, 0.5000, 0.3214]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 16.5675) [0.0000, 0.0000, 0.3929]\n", - "PeriodicSite: O (1.7391, 1.2298, 19.5798) [0.5000, 0.5000, 0.3929]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 19.5798) [0.0000, 0.0000, 0.4643]\n", - "PeriodicSite: O (1.7391, 1.2298, 22.5921) [0.5000, 0.5000, 0.4643]]\n", + "PeriodicSite: Mg (0.0, 0.0, 1.506) [0.0, 0.0, 0.03571]\n", + "PeriodicSite: O (1.739, 1.23, 4.518) [0.5, 0.5, 0.03571]\n", + "PeriodicSite: Mg (0.0, 0.0, 4.518) [0.0, 0.0, 0.1071]\n", + "PeriodicSite: O (1.739, 1.23, 7.531) [0.5, 0.5, 0.1071]\n", + "PeriodicSite: Mg (0.0, 0.0, 7.531) [0.0, 0.0, 0.1786]\n", + "PeriodicSite: O (1.739, 1.23, 10.54) [0.5, 0.5, 0.1786]\n", + "PeriodicSite: Mg (0.0, 0.0, 10.54) [0.0, 0.0, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 13.56) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (0.0, 0.0, 13.56) [0.0, 0.0, 0.3214]\n", + "PeriodicSite: O (1.739, 1.23, 16.57) [0.5, 0.5, 0.3214]\n", + "PeriodicSite: Mg (0.0, 0.0, 16.57) [0.0, 0.0, 0.3929]\n", + "PeriodicSite: O (1.739, 1.23, 19.58) [0.5, 0.5, 0.3929]\n", + "PeriodicSite: Mg (0.0, 0.0, 19.58) [0.0, 0.0, 0.4643]\n", + "PeriodicSite: O (1.739, 1.23, 22.59) [0.5, 0.5, 0.4643]]\n", ":sublattice:relax_slabs(3): [Structure Summary\n", "Lattice\n", - " abc : 3.012274887854692 3.0122748878546917 30.122748878546922\n", - " angles : 120.00000000000001 120.00000000000001 60.00000000000001\n", - " volume : 193.27193999999986\n", - " A : 2.6087065760640837 0.0 -1.5061374439273467\n", - " B : 0.8695688586880268 2.459512146747805 -1.5061374439273467\n", + " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", + " angles : 120.00000000000001 120.00000000000001 59.99999999999999\n", + " volume : 193.27193999999994\n", + " A : 2.608706576064084 0.0 -1.5061374439273467\n", + " B : 0.8695688586880276 2.4595121467478056 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0808, -0.0571, 2.1193) [-0.0232, -0.0232, 0.0680]\n", - "PeriodicSite: O (1.8010, 1.2735, -0.6459) [0.5178, 0.5178, 0.0303]\n", - "PeriodicSite: Mg (-0.0401, -0.0284, 5.2020) [-0.0115, -0.0115, 0.1715]\n", - "PeriodicSite: O (1.7790, 1.2579, 2.3282) [0.5114, 0.5114, 0.1284]\n", - "PeriodicSite: Mg (-0.0392, -0.0277, 8.2159) [-0.0113, -0.0113, 0.2716]\n", - "PeriodicSite: O (1.7778, 1.2571, 5.3384) [0.5111, 0.5111, 0.2283]\n", - "PeriodicSite: Mg (-0.0320, -0.0226, 11.2406) [-0.0092, -0.0092, 0.3722]\n", - "PeriodicSite: O (1.7829, 1.2607, 8.3596) [0.5126, 0.5126, 0.3288]\n", - "PeriodicSite: Mg (-0.0059, -0.0042, 14.2980) [-0.0017, -0.0017, 0.4745]\n", - "PeriodicSite: O (1.7529, 1.2395, 11.3199) [0.5040, 0.5040, 0.4262], Structure Summary\n", + "PeriodicSite: Mg (-0.06335, -0.04479, 2.149) [-0.01821, -0.01821, 0.06954]\n", + "PeriodicSite: O (1.804, 1.275, -0.6412) [0.5186, 0.5186, 0.03057]\n", + "PeriodicSite: Mg (-0.04308, -0.03047, 5.197) [-0.01239, -0.01239, 0.1713]\n", + "PeriodicSite: O (1.762, 1.246, 2.298) [0.5064, 0.5064, 0.1269]\n", + "PeriodicSite: Mg (-0.03833, -0.0271, 8.217) [-0.01102, -0.01102, 0.2717]\n", + "PeriodicSite: O (1.776, 1.255, 5.335) [0.5105, 0.5105, 0.2281]\n", + "PeriodicSite: Mg (-0.03796, -0.02684, 11.23) [-0.01091, -0.01091, 0.3717]\n", + "PeriodicSite: O (1.788, 1.265, 8.369) [0.5141, 0.5141, 0.3292]\n", + "PeriodicSite: Mg (-0.01275, -0.009017, 14.29) [-0.003666, -0.003666, 0.4739]\n", + "PeriodicSite: O (1.762, 1.246, 11.34) [0.5066, 0.5066, 0.427], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", - " angles : 120.00000000000001 60.00000000000001 90.0\n", + " angles : 120.00000000000001 59.99999999999999 90.0\n", " volume : 193.2719399999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", - " B : 0.8695688586880286 2.4595121467478047 -1.5061374439273467\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", + " B : 0.8695688586880287 2.459512146747805 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0015, 0.0021, 1.5088) [-0.0009, 0.0009, 0.0502]\n", - "PeriodicSite: O (1.7403, 1.2281, 1.5042) [0.5007, 0.4993, 0.0499]\n", - "PeriodicSite: Mg (0.0016, -0.0022, 4.5157) [0.0009, -0.0009, 0.1498]\n", - "PeriodicSite: O (1.7373, 1.2323, 4.5215) [0.4990, 0.5010, 0.1502]\n", - "PeriodicSite: Mg (0.0000, -0.0000, 7.5307) [0.0000, -0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 7.5307) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (-0.0016, 0.0022, 10.5457) [-0.0009, 0.0009, 0.3502]\n", - "PeriodicSite: O (1.7409, 1.2272, 10.5399) [0.5010, 0.4990, 0.3498]\n", - "PeriodicSite: Mg (0.0015, -0.0021, 13.5526) [0.0009, -0.0009, 0.4498]\n", - "PeriodicSite: O (1.7380, 1.2314, 13.5572) [0.4993, 0.5007, 0.4501], Structure Summary\n", + "PeriodicSite: Mg (-0.003667, 0.005186, 1.512) [-0.002108, 0.002108, 0.05042]\n", + "PeriodicSite: O (1.742, 1.226, 1.502) [0.5015, 0.4985, 0.0497]\n", + "PeriodicSite: Mg (0.004204, -0.005944, 4.511) [0.002417, -0.002417, 0.1495]\n", + "PeriodicSite: O (1.737, 1.233, 4.523) [0.4985, 0.5015, 0.1503]\n", + "PeriodicSite: Mg (-2.373e-07, 1.452e-07, 7.531) [-1.106e-07, 5.904e-08, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 7.531) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004206, 0.005949, 10.55) [-0.002419, 0.002419, 0.3505]\n", + "PeriodicSite: O (1.742, 1.226, 10.54) [0.5015, 0.4985, 0.3497]\n", + "PeriodicSite: Mg (0.003665, -0.005183, 13.55) [0.002107, -0.002107, 0.4496]\n", + "PeriodicSite: O (1.737, 1.233, 13.56) [0.4985, 0.5015, 0.4503], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 48.196398205675074\n", - " angles : 30.000000000000014 60.00000000000002 73.22134511903965\n", + " angles : 30.000000000000014 59.99999999999999 73.22134511903964\n", " volume : 309.2351039999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : -0.869568858688028 2.4595121467478056 4.518412331782038\n", " C : 0.0 0.0 48.196398205675074\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0262, 0.0105, 2.2139) [0.0114, 0.0042, 0.0452]\n", - "PeriodicSite: O (0.9825, 1.0539, 3.5697) [0.5195, 0.4285, 0.0177]\n", - "PeriodicSite: Mg (0.0662, 0.2336, 5.1568) [0.0571, 0.0950, 0.0963]\n", - "PeriodicSite: O (0.8459, 1.1867, 6.8186) [0.4851, 0.4825, 0.0811]\n", - "PeriodicSite: Mg (0.0133, 0.0507, 8.2607) [0.0120, 0.0206, 0.1691]\n", - "PeriodicSite: O (0.8496, 1.1727, 9.8246) [0.4846, 0.4768, 0.1440]\n", - "PeriodicSite: Mg (0.0086, 0.0959, 11.2811) [0.0163, 0.0390, 0.2299]\n", - "PeriodicSite: O (0.8389, 1.1668, 12.8553) [0.4797, 0.4744, 0.2073]\n", - "PeriodicSite: Mg (0.0091, 0.0613, 14.2926) [0.0118, 0.0249, 0.2938]\n", - "PeriodicSite: O (0.8384, 1.1314, 15.8685) [0.4747, 0.4600, 0.2713]\n", - "PeriodicSite: Mg (0.0019, 0.0338, 17.3173) [0.0053, 0.0137, 0.3579]\n", - "PeriodicSite: O (0.8477, 1.1616, 18.8647) [0.4824, 0.4723, 0.3321]\n", - "PeriodicSite: Mg (0.0237, 0.0330, 20.2919) [0.0135, 0.0134, 0.4193]\n", - "PeriodicSite: O (0.8368, 1.0796, 21.8957) [0.4671, 0.4390, 0.3986]\n", - "PeriodicSite: Mg (-0.0806, 0.1485, 23.4847) [-0.0108, 0.0604, 0.4819]\n", - "PeriodicSite: O (0.8484, 1.2181, 24.8879) [0.4903, 0.4953, 0.4546], Structure Summary\n", + "PeriodicSite: Mg (0.1339, -0.01997, 2.027) [0.04861, -0.008121, 0.04131]\n", + "PeriodicSite: O (1.084, 1.0, 3.395) [0.5509, 0.4068, 0.01508]\n", + "PeriodicSite: Mg (0.1856, 0.2225, 4.95) [0.1013, 0.09047, 0.09106]\n", + "PeriodicSite: O (0.9559, 1.166, 6.628) [0.5245, 0.4742, 0.07668]\n", + "PeriodicSite: Mg (0.09665, 0.08853, 8.116) [0.04905, 0.036, 0.1635]\n", + "PeriodicSite: O (0.9216, 1.175, 9.7) [0.5125, 0.4777, 0.1405]\n", + "PeriodicSite: Mg (0.04673, 0.1156, 11.22) [0.03359, 0.04702, 0.2272]\n", + "PeriodicSite: O (0.8436, 1.125, 12.85) [0.4758, 0.4574, 0.2088]\n", + "PeriodicSite: Mg (-0.005373, 0.07126, 14.32) [0.007599, 0.02897, 0.2941]\n", + "PeriodicSite: O (0.7974, 1.125, 15.94) [0.4581, 0.4574, 0.2735]\n", + "PeriodicSite: Mg (-0.06382, 0.06227, 17.43) [-0.01603, 0.02532, 0.3598]\n", + "PeriodicSite: O (0.7609, 1.136, 19.01) [0.4456, 0.4618, 0.3373]\n", + "PeriodicSite: Mg (-0.07782, 0.05423, 20.47) [-0.02248, 0.02205, 0.4233]\n", + "PeriodicSite: O (0.7283, 1.092, 22.08) [0.4271, 0.4439, 0.4032]\n", + "PeriodicSite: Mg (-0.1775, 0.196, 23.65) [-0.04147, 0.07969, 0.4846]\n", + "PeriodicSite: O (0.727, 1.229, 25.1) [0.4452, 0.4996, 0.46], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 42.17184842996569\n", - " angles : 29.999999999999964 59.999999999999986 54.735610317245346\n", + " angles : 29.99999999999997 59.999999999999986 54.735610317245346\n", " volume : 270.58071599999977\n", " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : 0.8695688586880266 2.4595121467478043 4.51841233178204\n", " C : 0.0 0.0 42.17184842996569\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0254, -0.0718, 1.5501) [0.0000, -0.0292, 0.0399]\n", - "PeriodicSite: O (1.7306, 1.2057, 4.5332) [0.5000, 0.4902, 0.0371]\n", - "PeriodicSite: Mg (0.0279, 0.0790, 4.4700) [0.0000, 0.0321, 0.1026]\n", - "PeriodicSite: O (1.7433, 1.2415, 7.5235) [0.5000, 0.5048, 0.1065]\n", - "PeriodicSite: Mg (-0.0041, -0.0115, 7.5377) [0.0000, -0.0047, 0.1792]\n", - "PeriodicSite: O (1.7393, 1.2301, 10.5427) [0.5000, 0.5002, 0.1785]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 10.5430) [0.0000, 0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 13.5552) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (0.0041, 0.0115, 13.5482) [0.0000, 0.0047, 0.3208]\n", - "PeriodicSite: O (1.7390, 1.2294, 16.5678) [0.5000, 0.4998, 0.3215]\n", - "PeriodicSite: Mg (-0.0279, -0.0790, 16.6159) [0.0000, -0.0321, 0.3974]\n", - "PeriodicSite: O (1.7350, 1.2180, 19.5870) [0.5000, 0.4952, 0.3935]\n", - "PeriodicSite: Mg (0.0254, 0.0718, 19.5358) [-0.0000, 0.0292, 0.4601]\n", - "PeriodicSite: O (1.7477, 1.2539, 22.5773) [0.5000, 0.5098, 0.4629]]\n", - ":parameter:(4): \n", - ":parameter:{\"executor\": \"local\", \"workflow_executor\": \"dask\", \"deps\": {}, \"call_before\": [], \"call_after\": [], \"triggers\": null, \"executor_data\": {}, \"workflow_executor_data\": {}}(5): {\"executor\": \"local\", \"workflow_executor\": \"dask\", \"deps\": {}, \"call_before\": [], \"call_after\": [], \"triggers\": null, \"executor_data\": {}, \"workflow_executor_data\": {}}\n", - ":postprocess:(6): [Structure Summary\n", + "PeriodicSite: Mg (-0.02746, -0.07768, 1.554) [2.276e-07, -0.03158, 0.04023]\n", + "PeriodicSite: O (1.734, 1.216, 4.527) [0.5, 0.4945, 0.0365]\n", + "PeriodicSite: Mg (0.02554, 0.07223, 4.474) [1.088e-06, 0.02937, 0.1029]\n", + "PeriodicSite: O (1.746, 1.25, 7.518) [0.5, 0.5081, 0.106]\n", + "PeriodicSite: Mg (0.004413, 0.01248, 7.523) [1.247e-07, 0.005074, 0.1778]\n", + "PeriodicSite: O (1.736, 1.22, 10.55) [0.5, 0.4962, 0.1791]\n", + "PeriodicSite: Mg (5.245e-07, 2.488e-06, 10.54) [-1.362e-07, 1.012e-06, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 13.56) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004412, -0.01248, 13.56) [-5.311e-08, -0.005073, 0.3222]\n", + "PeriodicSite: O (1.742, 1.239, 16.56) [0.5, 0.5038, 0.3209]\n", + "PeriodicSite: Mg (-0.02554, -0.07224, 16.61) [-3.741e-08, -0.02937, 0.3971]\n", + "PeriodicSite: O (1.732, 1.21, 19.59) [0.5, 0.4919, 0.394]\n", + "PeriodicSite: Mg (0.02746, 0.07767, 19.53) [-2.13e-08, 0.03158, 0.4598]\n", + "PeriodicSite: O (1.744, 1.243, 22.58) [0.5, 0.5055, 0.4635]]\n", + ":parameter:(4): \n", + ":parameter:{\"executor\": \"dask\", \"workflow_executor\": \"dask\", \"hooks\": {}, \"executor_data\": {}, \"workflow_executor_data\": {}, \"qelectron_data_exists\": false}(5): {\"executor\": \"dask\", \"workflow_executor\": \"dask\", \"hooks\": {}, \"executor_data\": {}, \"workflow_executor_data\": {}, \"qelectron_data_exists\": false}\n", + ":postprocess:reconstruct(6): [Structure Summary\n", "Lattice\n", - " abc : 3.012274887854692 3.0122748878546917 30.122748878546922\n", - " angles : 120.00000000000001 120.00000000000001 60.00000000000001\n", - " volume : 193.27193999999986\n", - " A : 2.6087065760640837 0.0 -1.5061374439273467\n", - " B : 0.8695688586880268 2.459512146747805 -1.5061374439273467\n", + " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", + " angles : 120.00000000000001 120.00000000000001 59.99999999999999\n", + " volume : 193.27193999999994\n", + " A : 2.608706576064084 0.0 -1.5061374439273467\n", + " B : 0.8695688586880276 2.4595121467478056 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0808, -0.0571, 2.1193) [-0.0232, -0.0232, 0.0680]\n", - "PeriodicSite: O (1.8010, 1.2735, -0.6459) [0.5178, 0.5178, 0.0303]\n", - "PeriodicSite: Mg (-0.0401, -0.0284, 5.2020) [-0.0115, -0.0115, 0.1715]\n", - "PeriodicSite: O (1.7790, 1.2579, 2.3282) [0.5114, 0.5114, 0.1284]\n", - "PeriodicSite: Mg (-0.0392, -0.0277, 8.2159) [-0.0113, -0.0113, 0.2716]\n", - "PeriodicSite: O (1.7778, 1.2571, 5.3384) [0.5111, 0.5111, 0.2283]\n", - "PeriodicSite: Mg (-0.0320, -0.0226, 11.2406) [-0.0092, -0.0092, 0.3722]\n", - "PeriodicSite: O (1.7829, 1.2607, 8.3596) [0.5126, 0.5126, 0.3288]\n", - "PeriodicSite: Mg (-0.0059, -0.0042, 14.2980) [-0.0017, -0.0017, 0.4745]\n", - "PeriodicSite: O (1.7529, 1.2395, 11.3199) [0.5040, 0.5040, 0.4262], Structure Summary\n", + "PeriodicSite: Mg (-0.06335, -0.04479, 2.149) [-0.01821, -0.01821, 0.06954]\n", + "PeriodicSite: O (1.804, 1.275, -0.6412) [0.5186, 0.5186, 0.03057]\n", + "PeriodicSite: Mg (-0.04308, -0.03047, 5.197) [-0.01239, -0.01239, 0.1713]\n", + "PeriodicSite: O (1.762, 1.246, 2.298) [0.5064, 0.5064, 0.1269]\n", + "PeriodicSite: Mg (-0.03833, -0.0271, 8.217) [-0.01102, -0.01102, 0.2717]\n", + "PeriodicSite: O (1.776, 1.255, 5.335) [0.5105, 0.5105, 0.2281]\n", + "PeriodicSite: Mg (-0.03796, -0.02684, 11.23) [-0.01091, -0.01091, 0.3717]\n", + "PeriodicSite: O (1.788, 1.265, 8.369) [0.5141, 0.5141, 0.3292]\n", + "PeriodicSite: Mg (-0.01275, -0.009017, 14.29) [-0.003666, -0.003666, 0.4739]\n", + "PeriodicSite: O (1.762, 1.246, 11.34) [0.5066, 0.5066, 0.427], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 3.012274887854692 30.122748878546922\n", - " angles : 120.00000000000001 60.00000000000001 90.0\n", + " angles : 120.00000000000001 59.99999999999999 90.0\n", " volume : 193.2719399999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", - " B : 0.8695688586880286 2.4595121467478047 -1.5061374439273467\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", + " B : 0.8695688586880287 2.459512146747805 -1.5061374439273467\n", " C : 0.0 0.0 30.122748878546922\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0015, 0.0021, 1.5088) [-0.0009, 0.0009, 0.0502]\n", - "PeriodicSite: O (1.7403, 1.2281, 1.5042) [0.5007, 0.4993, 0.0499]\n", - "PeriodicSite: Mg (0.0016, -0.0022, 4.5157) [0.0009, -0.0009, 0.1498]\n", - "PeriodicSite: O (1.7373, 1.2323, 4.5215) [0.4990, 0.5010, 0.1502]\n", - "PeriodicSite: Mg (0.0000, -0.0000, 7.5307) [0.0000, -0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 7.5307) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (-0.0016, 0.0022, 10.5457) [-0.0009, 0.0009, 0.3502]\n", - "PeriodicSite: O (1.7409, 1.2272, 10.5399) [0.5010, 0.4990, 0.3498]\n", - "PeriodicSite: Mg (0.0015, -0.0021, 13.5526) [0.0009, -0.0009, 0.4498]\n", - "PeriodicSite: O (1.7380, 1.2314, 13.5572) [0.4993, 0.5007, 0.4501], Structure Summary\n", + "PeriodicSite: Mg (-0.003667, 0.005186, 1.512) [-0.002108, 0.002108, 0.05042]\n", + "PeriodicSite: O (1.742, 1.226, 1.502) [0.5015, 0.4985, 0.0497]\n", + "PeriodicSite: Mg (0.004204, -0.005944, 4.511) [0.002417, -0.002417, 0.1495]\n", + "PeriodicSite: O (1.737, 1.233, 4.523) [0.4985, 0.5015, 0.1503]\n", + "PeriodicSite: Mg (-2.373e-07, 1.452e-07, 7.531) [-1.106e-07, 5.904e-08, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 7.531) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004206, 0.005949, 10.55) [-0.002419, 0.002419, 0.3505]\n", + "PeriodicSite: O (1.742, 1.226, 10.54) [0.5015, 0.4985, 0.3497]\n", + "PeriodicSite: Mg (0.003665, -0.005183, 13.55) [0.002107, -0.002107, 0.4496]\n", + "PeriodicSite: O (1.737, 1.233, 13.56) [0.4985, 0.5015, 0.4503], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 48.196398205675074\n", - " angles : 30.000000000000014 60.00000000000002 73.22134511903965\n", + " angles : 30.000000000000014 59.99999999999999 73.22134511903964\n", " volume : 309.2351039999999\n", - " A : 2.6087065760640846 0.0 1.5061374439273456\n", + " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : -0.869568858688028 2.4595121467478056 4.518412331782038\n", " C : 0.0 0.0 48.196398205675074\n", " pbc : True True True\n", - "PeriodicSite: Mg (0.0262, 0.0105, 2.2139) [0.0114, 0.0042, 0.0452]\n", - "PeriodicSite: O (0.9825, 1.0539, 3.5697) [0.5195, 0.4285, 0.0177]\n", - "PeriodicSite: Mg (0.0662, 0.2336, 5.1568) [0.0571, 0.0950, 0.0963]\n", - "PeriodicSite: O (0.8459, 1.1867, 6.8186) [0.4851, 0.4825, 0.0811]\n", - "PeriodicSite: Mg (0.0133, 0.0507, 8.2607) [0.0120, 0.0206, 0.1691]\n", - "PeriodicSite: O (0.8496, 1.1727, 9.8246) [0.4846, 0.4768, 0.1440]\n", - "PeriodicSite: Mg (0.0086, 0.0959, 11.2811) [0.0163, 0.0390, 0.2299]\n", - "PeriodicSite: O (0.8389, 1.1668, 12.8553) [0.4797, 0.4744, 0.2073]\n", - "PeriodicSite: Mg (0.0091, 0.0613, 14.2926) [0.0118, 0.0249, 0.2938]\n", - "PeriodicSite: O (0.8384, 1.1314, 15.8685) [0.4747, 0.4600, 0.2713]\n", - "PeriodicSite: Mg (0.0019, 0.0338, 17.3173) [0.0053, 0.0137, 0.3579]\n", - "PeriodicSite: O (0.8477, 1.1616, 18.8647) [0.4824, 0.4723, 0.3321]\n", - "PeriodicSite: Mg (0.0237, 0.0330, 20.2919) [0.0135, 0.0134, 0.4193]\n", - "PeriodicSite: O (0.8368, 1.0796, 21.8957) [0.4671, 0.4390, 0.3986]\n", - "PeriodicSite: Mg (-0.0806, 0.1485, 23.4847) [-0.0108, 0.0604, 0.4819]\n", - "PeriodicSite: O (0.8484, 1.2181, 24.8879) [0.4903, 0.4953, 0.4546], Structure Summary\n", + "PeriodicSite: Mg (0.1339, -0.01997, 2.027) [0.04861, -0.008121, 0.04131]\n", + "PeriodicSite: O (1.084, 1.0, 3.395) [0.5509, 0.4068, 0.01508]\n", + "PeriodicSite: Mg (0.1856, 0.2225, 4.95) [0.1013, 0.09047, 0.09106]\n", + "PeriodicSite: O (0.9559, 1.166, 6.628) [0.5245, 0.4742, 0.07668]\n", + "PeriodicSite: Mg (0.09665, 0.08853, 8.116) [0.04905, 0.036, 0.1635]\n", + "PeriodicSite: O (0.9216, 1.175, 9.7) [0.5125, 0.4777, 0.1405]\n", + "PeriodicSite: Mg (0.04673, 0.1156, 11.22) [0.03359, 0.04702, 0.2272]\n", + "PeriodicSite: O (0.8436, 1.125, 12.85) [0.4758, 0.4574, 0.2088]\n", + "PeriodicSite: Mg (-0.005373, 0.07126, 14.32) [0.007599, 0.02897, 0.2941]\n", + "PeriodicSite: O (0.7974, 1.125, 15.94) [0.4581, 0.4574, 0.2735]\n", + "PeriodicSite: Mg (-0.06382, 0.06227, 17.43) [-0.01603, 0.02532, 0.3598]\n", + "PeriodicSite: O (0.7609, 1.136, 19.01) [0.4456, 0.4618, 0.3373]\n", + "PeriodicSite: Mg (-0.07782, 0.05423, 20.47) [-0.02248, 0.02205, 0.4233]\n", + "PeriodicSite: O (0.7283, 1.092, 22.08) [0.4271, 0.4439, 0.4032]\n", + "PeriodicSite: Mg (-0.1775, 0.196, 23.65) [-0.04147, 0.07969, 0.4846]\n", + "PeriodicSite: O (0.727, 1.229, 25.1) [0.4452, 0.4996, 0.46], Structure Summary\n", "Lattice\n", " abc : 3.012274887854692 5.217413152128169 42.17184842996569\n", - " angles : 29.999999999999964 59.999999999999986 54.735610317245346\n", + " angles : 29.99999999999997 59.999999999999986 54.735610317245346\n", " volume : 270.58071599999977\n", " A : 2.608706576064084 0.0 1.5061374439273465\n", " B : 0.8695688586880266 2.4595121467478043 4.51841233178204\n", " C : 0.0 0.0 42.17184842996569\n", " pbc : True True True\n", - "PeriodicSite: Mg (-0.0254, -0.0718, 1.5501) [0.0000, -0.0292, 0.0399]\n", - "PeriodicSite: O (1.7306, 1.2057, 4.5332) [0.5000, 0.4902, 0.0371]\n", - "PeriodicSite: Mg (0.0279, 0.0790, 4.4700) [0.0000, 0.0321, 0.1026]\n", - "PeriodicSite: O (1.7433, 1.2415, 7.5235) [0.5000, 0.5048, 0.1065]\n", - "PeriodicSite: Mg (-0.0041, -0.0115, 7.5377) [0.0000, -0.0047, 0.1792]\n", - "PeriodicSite: O (1.7393, 1.2301, 10.5427) [0.5000, 0.5002, 0.1785]\n", - "PeriodicSite: Mg (0.0000, 0.0000, 10.5430) [0.0000, 0.0000, 0.2500]\n", - "PeriodicSite: O (1.7391, 1.2298, 13.5552) [0.5000, 0.5000, 0.2500]\n", - "PeriodicSite: Mg (0.0041, 0.0115, 13.5482) [0.0000, 0.0047, 0.3208]\n", - "PeriodicSite: O (1.7390, 1.2294, 16.5678) [0.5000, 0.4998, 0.3215]\n", - "PeriodicSite: Mg (-0.0279, -0.0790, 16.6159) [0.0000, -0.0321, 0.3974]\n", - "PeriodicSite: O (1.7350, 1.2180, 19.5870) [0.5000, 0.4952, 0.3935]\n", - "PeriodicSite: Mg (0.0254, 0.0718, 19.5358) [-0.0000, 0.0292, 0.4601]\n", - "PeriodicSite: O (1.7477, 1.2539, 22.5773) [0.5000, 0.5098, 0.4629]]\n", + "PeriodicSite: Mg (-0.02746, -0.07768, 1.554) [2.276e-07, -0.03158, 0.04023]\n", + "PeriodicSite: O (1.734, 1.216, 4.527) [0.5, 0.4945, 0.0365]\n", + "PeriodicSite: Mg (0.02554, 0.07223, 4.474) [1.088e-06, 0.02937, 0.1029]\n", + "PeriodicSite: O (1.746, 1.25, 7.518) [0.5, 0.5081, 0.106]\n", + "PeriodicSite: Mg (0.004413, 0.01248, 7.523) [1.247e-07, 0.005074, 0.1778]\n", + "PeriodicSite: O (1.736, 1.22, 10.55) [0.5, 0.4962, 0.1791]\n", + "PeriodicSite: Mg (5.245e-07, 2.488e-06, 10.54) [-1.362e-07, 1.012e-06, 0.25]\n", + "PeriodicSite: O (1.739, 1.23, 13.56) [0.5, 0.5, 0.25]\n", + "PeriodicSite: Mg (-0.004412, -0.01248, 13.56) [-5.311e-08, -0.005073, 0.3222]\n", + "PeriodicSite: O (1.742, 1.239, 16.56) [0.5, 0.5038, 0.3209]\n", + "PeriodicSite: Mg (-0.02554, -0.07224, 16.61) [-3.741e-08, -0.02937, 0.3971]\n", + "PeriodicSite: O (1.732, 1.21, 19.59) [0.5, 0.4919, 0.394]\n", + "PeriodicSite: Mg (0.02746, 0.07767, 19.53) [-2.13e-08, 0.03158, 0.4598]\n", + "PeriodicSite: O (1.744, 1.243, 22.58) [0.5, 0.5055, 0.4635]]\n", "\n" ] } @@ -656,7 +668,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/doc/source/tutorials/3_QuantumChemistry/nitrogen_copper_interaction/source.ipynb b/doc/source/tutorials/3_QuantumChemistry/nitrogen_copper_interaction/source.ipynb index 2f5b2c20c..aedff5159 100644 --- a/doc/source/tutorials/3_QuantumChemistry/nitrogen_copper_interaction/source.ipynb +++ b/doc/source/tutorials/3_QuantumChemistry/nitrogen_copper_interaction/source.ipynb @@ -36,6 +36,13 @@ " print(line.rstrip())\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You might have to install `numpy==1.23` again manually due to a conflict with `ase-notebook` package." + ] + }, { "cell_type": "code", "execution_count": 2, @@ -559,7 +566,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.18" }, "vscode": { "interpreter": { diff --git a/doc/source/tutorials/tutorials.rst b/doc/source/tutorials/tutorials.rst index e426f1cca..d026a7174 100644 --- a/doc/source/tutorials/tutorials.rst +++ b/doc/source/tutorials/tutorials.rst @@ -75,6 +75,8 @@ Advanced - :doc:`Scalable API backends for LLM and generative AI <./0_ClassicalMachineLearning/genai/source>` * - Federated learning - :doc:`Federated learning <./federated_learning/source>` + * - Voice cloning + - :doc:`Voice cloning <./voice_cloning/source>` --------------------------------- diff --git a/doc/source/tutorials/voice_cloning/Dockerfile_gcp b/doc/source/tutorials/voice_cloning/Dockerfile_gcp new file mode 100644 index 000000000..01fc77aad --- /dev/null +++ b/doc/source/tutorials/voice_cloning/Dockerfile_gcp @@ -0,0 +1,43 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the GNU Affero General Public License 3.0 (the "License"). +# A copy of the License may be obtained with this software package or at +# +# https://www.gnu.org/licenses/agpl-3.0.en.html +# +# Use of this file is prohibited except in compliance with the License. Any +# modifications or derivative works of this file must retain this copyright +# notice, and modified files must contain a notice indicating that they have +# been altered from the originals. +# +# Covalent is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the License for more details. +# +# Relief from the License may be granted by purchasing a commercial license. + +ARG COVALENT_BASE_IMAGE=python:3.9-slim-buster +FROM ${COVALENT_BASE_IMAGE} + +# Install dependencies +ARG COVALENT_TASK_ROOT=/covalent +WORKDIR ${COVALENT_TASK_ROOT} + +ARG COVALENT_PACKAGE_VERSION=0.229.0rc0 +ARG PRE_RELEASE="" + +COPY requirements.txt requirements.txt +RUN apt-get update && \ + apt-get install -y build-essential wget && \ + pip install -r requirements.txt + +COPY covalent_gcpbatch_plugin/exec.py ${COVALENT_TASK_ROOT} + +ENV PYTHONPATH ${COVALENT_TASK_ROOT}:${PYTHONPATH} + +# Path where the storage bucket will be mounted inside the container +ENV GCPBATCH_TASK_MOUNTPOINT /mnt/disks/covalent + +ENTRYPOINT [ "python", "exec.py" ] diff --git a/doc/source/tutorials/voice_cloning/assets/streamlit_gcp.gif b/doc/source/tutorials/voice_cloning/assets/streamlit_gcp.gif new file mode 100644 index 000000000..bc3691a7e Binary files /dev/null and b/doc/source/tutorials/voice_cloning/assets/streamlit_gcp.gif differ diff --git a/doc/source/tutorials/voice_cloning/assets/workflow.gif b/doc/source/tutorials/voice_cloning/assets/workflow.gif new file mode 100644 index 000000000..283fb35e4 Binary files /dev/null and b/doc/source/tutorials/voice_cloning/assets/workflow.gif differ diff --git a/doc/source/tutorials/voice_cloning/requirements.txt b/doc/source/tutorials/voice_cloning/requirements.txt new file mode 100644 index 000000000..534f3b8e0 --- /dev/null +++ b/doc/source/tutorials/voice_cloning/requirements.txt @@ -0,0 +1,12 @@ +covalent==0.229.0rc0 +covalent-gcpbatch-plugin==0.11.0 +librosa==0.10.0 +pydub==0.25.1 +pytube==15.0.0 +scipy==1.11.3 +soundfile==0.12.1 +streamlit==1.28.1 +torch==2.1.0 +torchaudio==2.1.0 +transformers==4.33.3 +TTS==0.19.1 diff --git a/doc/source/tutorials/voice_cloning/source.ipynb b/doc/source/tutorials/voice_cloning/source.ipynb new file mode 100644 index 000000000..754c998da --- /dev/null +++ b/doc/source/tutorials/voice_cloning/source.ipynb @@ -0,0 +1,401 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Youtube Video Summarization with Voice Cloning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "Our script performs several tasks:\n", + "\n", + "1. downloads and processes a YouTube video,\n", + "2. transcribes the audio from the YouTube video,\n", + "3. summarizes the transcription of the transcribed audio, and\n", + "4. converts the summary to speech using the user's voice.\n", + "\n", + "The script leverages Covalent for executing these tasks, either locally or on a cloud platform like GCP.\n", + "\n", + "## Import Dependencies\n", + "\n", + "First, we import necessary Python libraries:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import streamlit as st\n", + "\n", + "from pytube import YouTube\n", + "from pydub import AudioSegment\n", + "from transformers import pipeline\n", + "from TTS.api import TTS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This section loads libraries for audio processing, machine learning models, and Covalent for workflow management.\n", + "\n", + "```bash\n", + "covalent deploy up gcpbatch\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set Up Covalent and Dependencies\n", + "Covalent simplifies cloud resource management. We define dependencies for each task and configure a Covalent executor for cloud execution. \n", + "In the example below we utilize [Google Cloud Batch](https://cloud.google.com/batch) using our [gcp batch executor](https://docs.covalent.xyz/docs/user-documentation/api-reference/executors/gcp/). " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "audio_deps = [\n", + " \"transformers==4.33.3\", \"pydub==0.25.1\",\n", + " \"torchaudio==2.1.0\", \"librosa==0.10.0\",\n", + " \"torch==2.1.0\"\n", + "]\n", + "text_deps = [\"transformers==4.33.3\", \"torch==2.1.0\"]\n", + "tts_deps = audio_deps + [\"TTS==0.19.1\"]\n", + "\n", + "executor = ct.executor.GCPBatchExecutor(\n", + " container_image_uri=\"docker.io/filipbolt/covalent-gcp-0.229.0rc0\",\n", + " vcpus=4,\n", + " memory=8192,\n", + " time_limit=3000,\n", + " poll_freq=1,\n", + " retries=1\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you may use Covalent Cloud to execute this workflow by doing:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import covalent_cloud as cc\n", + "\n", + "cc_executor = cc.CloudExecutor(num_cpus=4, env=\"genai-env\", memory=8192, time_limit=3000)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Covalent Tasks\n", + "Each step of our workflow is encapsulated in a Covalent task. Here's an example:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@ct.electron\n", + "def download_video(url):\n", + " yt = YouTube(url)\n", + " # download file\n", + " out_file = yt.streams.filter(\n", + " only_audio=True, file_extension=\"mp4\"\n", + " ).first().download(\".\")\n", + "\n", + " # rename downloaded file\n", + " os.rename(out_file, \"audio.mp4\")\n", + " with open(\"audio.mp4\", \"rb\") as f:\n", + " file_content = f.read()\n", + " return file_content\n", + "\n", + "\n", + "@ct.electron\n", + "def load_audio(input_file_content):\n", + " input_path = os.path.join(os.getcwd(), \"file.mp4\")\n", + " # write to file\n", + " with open(input_path, \"wb\") as f:\n", + " f.write(input_file_content)\n", + "\n", + " audio_content = AudioSegment.from_file(input_path, format=\"mp4\")\n", + " return audio_content\n", + "\n", + "\n", + "@ct.electron(executor=executor, deps_pip=audio_deps)\n", + "def transcribe_audio(audio_content):\n", + " # Export the audio as a WAV file\n", + " audio_content.export(\"audio_file.wav\", format=\"wav\")\n", + "\n", + " pipe = pipeline(\n", + " task=\"automatic-speech-recognition\",\n", + " # model=\"openai/whisper-small\",\n", + " model=\"openai/whisper-large-v3\",\n", + " chunk_length_s=30, max_new_tokens=2048,\n", + " )\n", + " transcription = pipe(\"audio_file.wav\")\n", + " return transcription['text']\n", + "\n", + "\n", + "@ct.electron(executor=executor, deps_pip=text_deps)\n", + "def summarize_transcription(transcription):\n", + " summarizer = pipeline(\n", + " \"summarization\",\n", + " model=\"facebook/bart-large-cnn\",\n", + " )\n", + " summary = summarizer(\n", + " transcription, min_length=5, max_length=100,\n", + " do_sample=False, truncation=True\n", + " )[0][\"summary_text\"]\n", + " return summary\n", + "\n", + "\n", + "@ct.electron(executor=executor, deps_pip=tts_deps)\n", + "def text_to_speech_voice_clone(text, speaker_content, output_file):\n", + " with open(\"speaker.wav\", \"wb\") as f:\n", + " f.write(speaker_content)\n", + "\n", + " # agree to service agreement programmatically\n", + " os.environ['COQUI_TOS_AGREED'] = \"1\"\n", + "\n", + " tts = TTS(\"tts_models/multilingual/multi-dataset/xtts_v1\")\n", + " tts.tts_to_file(\n", + " text=text,\n", + " file_path=output_file,\n", + " speaker_wav=\"speaker.wav\",\n", + " language=\"en\"\n", + " )\n", + " with open(output_file, \"rb\") as f:\n", + " file_content = f.read()\n", + " return file_content\n", + "\n", + "\n", + "@ct.electron\n", + "def load_wav_file(wav_file):\n", + " with open(wav_file, \"rb\") as f:\n", + " file_content = f.read()\n", + " return file_content" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use the `@ct.electron` decorator to define tasks like `download_video`,`mp4_to_wav`, `transcribe_audio`, etc." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Orchestrate the Workflow\n", + "The `@ct.lattice` decorator is used to define the workflow that orchestrates the entire process:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "@ct.lattice\n", + "def workflow(url, user_voice_file, output_file):\n", + " video_content = download_video(url)\n", + " audio_content = load_audio(video_content)\n", + "\n", + " user_voice_content = load_wav_file(user_voice_file)\n", + "\n", + " # Use Google Cloud Batch to transcribe, summarize and re-voice\n", + " transcription = transcribe_audio(audio_content)\n", + " summary = summarize_transcription(transcription)\n", + " output_file_content = text_to_speech_voice_clone(\n", + " summary, user_voice_content, output_file\n", + " )\n", + " return summary, transcription, output_file_content" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streamlit Interface\n", + "We use Streamlit to create an interactive web interface for the script:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-01-08 14:37:20.421 \n", + " \u001b[33m\u001b[1mWarning:\u001b[0m to view this Streamlit app on a browser, run it with the following\n", + " command:\n", + "\n", + " streamlit run /home/filip/miniconda3/envs/google/lib/python3.9/site-packages/ipykernel_launcher.py [ARGUMENTS]\n", + "2024-01-08 14:37:20.422 Session state does not function when running a script without `streamlit run`\n" + ] + } + ], + "source": [ + "import streamlit as st\n", + "\n", + "# Function to display results\n", + "def display_results(summary, transcription, audio_file_content):\n", + " display_summary(summary)\n", + " display_full_transcription(transcription)\n", + " display_audio_summary(audio_file_content)\n", + "\n", + "# Function to display the summary\n", + "def display_summary(summary):\n", + " st.subheader(\"YouTube transcription summary:\")\n", + " st.text(summary)\n", + "\n", + "# Function to display the full transcription with a toggle\n", + "def display_full_transcription(transcription):\n", + " st.subheader(\"YouTube full transcription\")\n", + " if st.checkbox(\"Show/Hide\", False):\n", + " st.text(transcription)\n", + "\n", + "# Function to display the audio summary\n", + "def display_audio_summary(audio_file_content):\n", + " st.subheader(\"Summary in your own voice:\")\n", + " st.audio(audio_file_content, format=\"audio/wav\")\n", + "\n", + "\n", + "# Streamlit app layout\n", + "def main():\n", + " st.title(\"Summarize YouTube videos in your own voice using AI\")\n", + " speaker_file, speaker_file_path = upload_speaker_file()\n", + " youtube_url = st.text_input(\"Enter valid YouTube URL\")\n", + "\n", + " if st.button(\"Process\"):\n", + " process_input(speaker_file, speaker_file_path, youtube_url)\n", + " elif \"transcription\" in st.session_state:\n", + " display_results(\n", + " st.session_state[\"summary\"],\n", + " st.session_state[\"transcription\"],\n", + " st.session_state[\"audio_file_content\"]\n", + " )\n", + "\n", + "# Function to upload speaker file\n", + "def upload_speaker_file():\n", + " speaker_file = st.file_uploader(\"Upload an audio file (WAV)\", type=[\"wav\"])\n", + " if speaker_file:\n", + " st.audio(speaker_file, format=\"audio/wav\")\n", + " speaker_file_path = \"speaker.wav\"\n", + " with open(speaker_file_path, \"wb\") as f:\n", + " f.write(speaker_file.getbuffer())\n", + " return speaker_file, speaker_file_path\n", + " return None, None\n", + "\n", + "# Function to process the input\n", + "def process_input(speaker_file, speaker_file_path, youtube_url):\n", + " if speaker_file and youtube_url:\n", + " audio_file_full_path = os.path.join(os.getcwd(), \"audio.wav\")\n", + " speaker_file_full_path = os.path.join(os.getcwd(), speaker_file_path)\n", + "\n", + " dispatch_id = ct.dispatch(workflow)(\n", + " youtube_url, speaker_file_full_path, audio_file_full_path\n", + " )\n", + " with st.spinner(f\"Processing... job dispatch id: {dispatch_id}\"):\n", + " result = ct.get_result(dispatch_id, wait=True)\n", + "\n", + " if result:\n", + " summary, transcription, output_file_content = result.result\n", + " st.session_state[\"transcription\"] = transcription\n", + " st.session_state[\"summary\"] = summary\n", + " st.session_state[\"audio_file_content\"] = output_file_content\n", + " display_results(summary, transcription, output_file_content)\n", + " else:\n", + " st.error(\"Something went wrong. Please try again.\")\n", + "\n", + "main()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Running the Script\n", + "To run the script, execute it via Streamlit:\n", + "\n", + "```bash\n", + "streamlit run your_script.py\n", + "```\n", + "\n", + "In the Covalent UI, you should be seeing a workflow like the following\n", + "\n", + "![alt text](assets/workflow.gif)\n", + "\n", + "The streamlit app usage would then be:\n", + "\n", + "![alt text](assets/streamlit_gcp.gif)\n", + "\n", + "### Customizing the Workflow\n", + "You can tailor this script to your specific needs:\n", + "\n", + "- Modify the Covalent task functions for different processing requirements like swapping one of the models.\n", + "- Adjust the Covalent executor settings based on your cloud resource needs.\n", + "\n", + "### Conclusion\n", + "\n", + "This tutorial demonstrates using Covalent for fine-tuning a speech summarization model. Covalent's cloud computing abstraction simplifies executing complex workflows, making it a powerful tool for developers and researchers in AI/ML fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements-qelectron.txt b/requirements-qelectron.txt new file mode 100644 index 000000000..98b537b50 --- /dev/null +++ b/requirements-qelectron.txt @@ -0,0 +1,4 @@ +lmdbm>=0.0.5 +mpire>=2.7.1 +orjson>=3.8.10 +pennylane>=0.31.1,<0.33.0 diff --git a/requirements.txt b/requirements.txt index bdffc7147..72f23b837 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,12 +8,8 @@ dask[distributed]>=2022.6.0 fastapi>=0.100.0 filelock>=3.12.2 furl>=2.1.3 -lmdbm>=0.0.5 -mpire>=2.7.1 natsort>=8.4.0 networkx>=2.8.6 -orjson>=3.8.10 -pennylane>=0.31.1,<0.33.0 psutil>=5.9.0 pydantic>=2.1.1 python-multipart>=0.0.6 diff --git a/setup.py b/setup.py index 50e3d6b63..317a469c7 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,11 @@ with open(requirements_file) as f: required = f.read().splitlines() +# By default we don't install qelectron requirements +# and only install them as an extra +with open("requirements-qelectron.txt") as f: + qelectron_reqs = f.read().splitlines() + def recursively_append_files(directory: str): """ @@ -220,6 +225,7 @@ def find_sources(self): "qiskit-ibm-provider==0.6.1", "qiskit-ibm-runtime==0.10.0", ], + "quantum": qelectron_reqs, }, "classifiers": [ "Development Status :: 4 - Beta", diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py index ec02aa3f0..a50b083da 100644 --- a/tests/covalent_dispatcher_tests/_cli/cli_test.py +++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py @@ -184,22 +184,14 @@ def test_deploy_up(mocker): "covalent_dispatcher._cli.groups.deploy_group._run_command_and_show_output", ) - # Fail with invalid command options. - mocker.patch( - "covalent_dispatcher._cli.groups.deploy_group.validate_args", - return_value="Non-empty msg", - ) + # Fail with invalid executor name with pytest.raises(SystemExit) as exc_info: ctx = click.Context(up) - ctx.invoke(up) + ctx.invoke(up, executor_name="invalid") assert exc_info.value.code == 1 # Succeed but exit after help message. - mocker.patch( - "covalent_dispatcher._cli.groups.deploy_group.validate_args", - return_value=None, - ) mocker.patch( "covalent_dispatcher._cli.groups.deploy_group.get_crm_object", ) @@ -227,6 +219,15 @@ def test_deploy_down(mocker): mock_run_command_and_show_output = mocker.patch( "covalent_dispatcher._cli.groups.deploy_group._run_command_and_show_output", ) + + # Fail with invalid executor name + with pytest.raises(SystemExit) as exc_info: + ctx = click.Context(down) + ctx.invoke(down, executor_name="invalid") + + assert exc_info.value.code == 1 + + # Succeed with valid command options. mocker.patch( "covalent_dispatcher._cli.groups.deploy_group.get_crm_object", ) diff --git a/tests/covalent_tests/results_manager_tests/results_manager_test.py b/tests/covalent_tests/results_manager_tests/results_manager_test.py index f7bb829fb..06ba8ece0 100644 --- a/tests/covalent_tests/results_manager_tests/results_manager_test.py +++ b/tests/covalent_tests/results_manager_tests/results_manager_test.py @@ -261,6 +261,19 @@ def test_get_result_404(mocker): get_result(dispatch_id) +def test_get_result_RecursionError(mocker): + """Check exception handing for RecursionError.""" + + dispatch_id = "test_get_result_RecursionError" + + with pytest.raises(RuntimeError): + with mocker.patch( + "covalent._results_manager.results_manager._get_result_multistage", + side_effect=RecursionError("from test_get_result_RecursionError"), + ): + get_result(dispatch_id, wait=True) + + def test_get_status_only(mocker): """Check get_result when status_only=True""" diff --git a/tests/covalent_tests/shared_files/qelectron_utils_test.py b/tests/covalent_tests/shared_files/qelectron_utils_test.py index 6c213e217..2cb66dbb6 100644 --- a/tests/covalent_tests/shared_files/qelectron_utils_test.py +++ b/tests/covalent_tests/shared_files/qelectron_utils_test.py @@ -30,7 +30,7 @@ def test_get_qelectron_db_path(mocker, db_exists): """Test the get_qelectron_db_path function.""" - mock_database = mocker.patch("covalent._shared_files.qelectron_utils.Database") + mock_database = mocker.patch("covalent.quantum.qserver.database.Database") mock_database.return_value.get_db_path.return_value.exists.return_value = db_exists dispatch_id = "mock_dispatch_id" diff --git a/tests/functional_tests/file_transfer_test.py b/tests/functional_tests/file_transfer_test.py index ea5f4029d..36d9ff2ac 100644 --- a/tests/functional_tests/file_transfer_test.py +++ b/tests/functional_tests/file_transfer_test.py @@ -180,7 +180,7 @@ def workflow(): rm._delete_result(dispatch_id) assert len(workflow_result.result) == 2 - assert workflow_result.result == (MOCK_CONTENTS, MOCK_CONTENTS) + assert workflow_result.result == [MOCK_CONTENTS, MOCK_CONTENTS] for f in [source_file_1, dest_file_1, source_file_2, dest_file_2]: f.unlink() diff --git a/tests/qelectron_tests/core_tests/__init__.py b/tests/qelectron_tests/core_tests/__init__.py new file mode 100644 index 000000000..6b2415372 --- /dev/null +++ b/tests/qelectron_tests/core_tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/qelectron_tests/test_decorator.py b/tests/qelectron_tests/core_tests/test_decorator.py similarity index 98% rename from tests/qelectron_tests/test_decorator.py rename to tests/qelectron_tests/core_tests/test_decorator.py index f376d0099..3bd11da15 100644 --- a/tests/qelectron_tests/test_decorator.py +++ b/tests/qelectron_tests/core_tests/test_decorator.py @@ -25,7 +25,7 @@ import covalent as ct EXECUTORS = [ - ct.executor.QiskitExecutor(device="local_sampler", shots=10_000), + ct.executor.Simulator(), ] diff --git a/tests/qelectron_tests/test_qelectron_db.py b/tests/qelectron_tests/core_tests/test_qelectron_db.py similarity index 96% rename from tests/qelectron_tests/test_qelectron_db.py rename to tests/qelectron_tests/core_tests/test_qelectron_db.py index 4d75f296e..7bc771003 100644 --- a/tests/qelectron_tests/test_qelectron_db.py +++ b/tests/qelectron_tests/core_tests/test_qelectron_db.py @@ -25,7 +25,7 @@ def test_db_exposed_in_result(): """ # Define a QElectron circuit. - qexecutor = ct.executor.QiskitExecutor(device="local_sampler") # pylint: disable=no-member + qexecutor = ct.executor.Simulator() # pylint: disable=no-member @ct.qelectron(executors=qexecutor) @qml.qnode(qml.device("default.qubit", wires=1)) diff --git a/tests/qelectron_tests/test_run_later.py b/tests/qelectron_tests/core_tests/test_run_later.py similarity index 97% rename from tests/qelectron_tests/test_run_later.py rename to tests/qelectron_tests/core_tests/test_run_later.py index 169e96662..aec023182 100644 --- a/tests/qelectron_tests/test_run_later.py +++ b/tests/qelectron_tests/core_tests/test_run_later.py @@ -23,7 +23,6 @@ import covalent as ct EXECUTORS = [ - ct.executor.QiskitExecutor(device="local_sampler", shots=10_000), ct.executor.Simulator(), ] diff --git a/tests/qelectron_tests/pennylane_tests/conftest.py b/tests/qelectron_tests/pennylane_tests/conftest.py index 30b774570..773ba2edf 100644 --- a/tests/qelectron_tests/pennylane_tests/conftest.py +++ b/tests/qelectron_tests/pennylane_tests/conftest.py @@ -33,7 +33,7 @@ import pytest import covalent as ct -from covalent._shared_files.utils import get_original_shots +from covalent._shared_files.qelectron_utils import get_original_shots from covalent.quantum.qcluster.simulator import SIMULATOR_DEVICES SKIP_RETURN_TYPES = ["qml.apply", "qml.vn_entropy", "qml.mutual_info"] diff --git a/tests/qelectron_tests/plugin_tests/__init__.py b/tests/qelectron_tests/plugin_tests/__init__.py new file mode 100644 index 000000000..6b2415372 --- /dev/null +++ b/tests/qelectron_tests/plugin_tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2024 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/qelectron_tests/test_braket_plugin.py b/tests/qelectron_tests/plugin_tests/test_braket_plugin.py similarity index 100% rename from tests/qelectron_tests/test_braket_plugin.py rename to tests/qelectron_tests/plugin_tests/test_braket_plugin.py diff --git a/tests/qelectron_tests/test_qiskit_plugin.py b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py similarity index 98% rename from tests/qelectron_tests/test_qiskit_plugin.py rename to tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py index 438532424..b6046005f 100644 --- a/tests/qelectron_tests/test_qiskit_plugin.py +++ b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin.py @@ -23,7 +23,7 @@ import covalent as ct from covalent._shared_files.config import get_config -from .utils import arg_vector, simple_circuit, weight_vector +from ..utils import arg_vector, simple_circuit, weight_vector EXECUTOR_CLASSES = [ ct.executor.QiskitExecutor, diff --git a/tests/qelectron_tests/test_qiskit_plugin_runtime.py b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py similarity index 98% rename from tests/qelectron_tests/test_qiskit_plugin_runtime.py rename to tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py index 913d54388..e21724d23 100644 --- a/tests/qelectron_tests/test_qiskit_plugin_runtime.py +++ b/tests/qelectron_tests/plugin_tests/test_qiskit_plugin_runtime.py @@ -25,7 +25,7 @@ import covalent as ct from covalent._shared_files.config import get_config -from .utils import arg_vector, cyclic_selector, get_hamiltonian_circuit, weight_vector +from ..utils import arg_vector, cyclic_selector, get_hamiltonian_circuit, weight_vector EXECUTOR_CLASSES = [ ct.executor.QiskitExecutor, diff --git a/tests/requirements.txt b/tests/requirements.txt index 99f936c73..38c17dc3c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,6 +5,7 @@ isort>=5.10.1 locust>=2.11.0 mock>=4.0.3 nbconvert>=6.5.1 +pennylane>=0.31.1 pre-commit>=2.20.0 pytest>=7.1.3 pytest-asyncio>=0.21.0