Enabling qelectron tests and making qelectron opt-in only (#1916)

* Adding qelectron tests to the suite

* disabling qelectron tests as they are not working currently

* making qelectrons a feature users can opt out of

* fixed basic dispatching

* fixing boilerplate and requirements workflows

* fixing boilerplate and requirements workflows

* fixing tests

* fixing tests and workflows

* fixing requirements

* fixing requirements

* Made qelectron an opt-in feature

* minor import fix

* fixing tests

* fixing tests

* added executor back to init file

.github/workflows/boilerplate.yml

@@ -85,9 +85,26 @@ jobs:
             # See the License for the specific language governing permissions and
             # limitations under the License.
 
+          boilerplate2024: |-
+            # Copyright 2024 Agnostiq Inc.
+            #
+            # This file is part of Covalent.
+            #
+            # Licensed under the Apache License 2.0 (the "License"). A copy of the
+            # License may be obtained with this software package or at
+            #
+            #     https://www.apache.org/licenses/LICENSE-2.0
+            #
+            # Use of this file is prohibited except in compliance with the License.
+            # Unless required by applicable law or agreed to in writing, software
+            # distributed under the License is distributed on an "AS IS" BASIS,
+            # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+            # See the License for the specific language governing permissions and
+            # limitations under the License.
+
         run: |
           for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
-            if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" ]] ; then
+            if [[ ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2021" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2022" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2023" && ! $( cat $file | tr -d '\r' ) =~ "$boilerplate2024" ]] ; then
               printf "Boilerplate is missing from $file.\n"
               printf "The first 15 lines of $file are\n\n"
               cat $file | tr -d '\r' | cat -ET | head -n 15

.github/workflows/requirements.yml

@@ -50,10 +50,14 @@ jobs:
           --ignore-module=pkg_resources
           --ignore-module=covalent/_dispatcher_plugins
           --ignore-module=covalent/_shared_files
+          --ignore-file=covalent/quantum/**
+          --ignore-file=covalent/_workflow/q*
+          --ignore-file=covalent/_shared_files/q*
+          --ignore-file=covalent/_results_manager/q*
+          --ignore-file=covalent/_shared_files/pickling.py
           --ignore-file=covalent/executor/**
           --ignore-file=covalent/triggers/**
           --ignore-file=covalent/cloud_resource_manager/**
-          --ignore-file=covalent/quantum/qserver/**
           --ignore-file=covalent/_programmatic/**
           covalent
 

.github/workflows/tests.yml

@@ -90,6 +90,11 @@ jobs:
             sdk:
             - 'covalent/**'
             - 'tests/covalent_tests/**'
+            qelectron:
+            - 'covalent/executor/quantum_plugins/**'
+            - 'covalent/executor/qbase.py'
+            - 'covalent/quantum/**'
+            - 'tests/qelectron_tests/**'
             dispatcher:
             - 'covalent_dispatcher/**'
             - 'tests/covalent_dispatcher_tests/**'
@@ -134,6 +139,7 @@ jobs:
           echo "NEED_PYTHON=$NEED_PYTHON" >> $GITHUB_ENV
           echo "NEED_FRONTEND=$NEED_FRONTEND" >> $GITHUB_ENV
           echo "BUILD_AND_RUN_ALL=$BUILD_AND_RUN_ALL" >> $GITHUB_ENV
+          echo "COVALENT_DISABLE_QELECTRON_TESTS=true" >> $GITHUB_ENV
 
       - name: Set up Python
         if: >
@@ -159,6 +165,7 @@ jobs:
         run: |
           pip install --no-cache-dir -r ./requirements.txt
           pip install --no-cache-dir -r ./tests/requirements.txt
+          pip install --no-cache-dir -r ./requirements-qelectron.txt
 
       - name: Set up Node
         if: env.NEED_FRONTEND || env.BUILD_AND_RUN_ALL
@@ -252,6 +259,18 @@ jobs:
         if: steps.sdk-tests.outcome == 'success'
         run: coverage xml -o sdk_coverage.xml
 
+      - name: Run Qelectron tests and measure coverage
+        id: qelectron-tests
+        if: >
+          (steps.modified-files.outputs.qelectron == 'true'
+          || env.BUILD_AND_RUN_ALL) && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true'
+        run: PYTHONPATH=$PWD/ pytest -vvs  --reruns=5 tests/qelectron_tests/core_tests --cov=covalent_qelectron --cov-config=.coveragerc
+
+      - name: Generate Qelectron coverage report
+        id: qelectron-coverage
+        if: steps.qelectron-tests.outcome == 'success' && env.COVALENT_DISABLE_QELECTRON_TESTS != 'true'
+        run: coverage xml -o qelectron_coverage.xml
+
       - name: Run dispatcher tests and measure coverage
         id: dispatcher-tests
         if: >

.gitignore

@@ -59,6 +59,7 @@
 !pyproject.toml
 !requirements.txt
 !requirements-client.txt
+!requirements-qelectron.txt
 !setup.py
 
 # Allow markdown etc

CHANGELOG.md

@@ -7,11 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Added
+
+- Added `pennylane` as a requirement in tests due to the tutorials using it
+
 ### Changed
 
 - Updated RTD notebooks to fix their behavior
 - Changed the error being shown when drawing the transport graph of a lattice to a debug message instead
 - Revamped README
+- Reorganized `qelectron` tests
+- Made qelectron an opt-in feature using `covalent[quantum]` extra
 
 ### Removed
 
@@ -26,8 +32,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Fixed the scenario where any deploy commands would fail if the user had a non deploy compatible plugin installed
+- Fixed the SQLAlchemy warning that used to show up at every fresh server start
 - Fixed deploy commands' default value of plugins not being propagated to the tfvars file
 
+### Operations
+
+- Added qelectron tests to the `tests` workflow
+
 ## [0.233.0-rc.0] - 2024-01-07
 
 ### Authors

MANIFEST.in

@@ -1,6 +1,7 @@
 include VERSION
 include requirements.txt
 include requirements-client.txt
+include requirements-qelectron.txt
 include covalent/py.typed
 recursive-include covalent/executor/ *
 recursive-include covalent_dispatcher/_service/ *

covalent/__init__.py

@@ -16,6 +16,7 @@
 
 """Main Covalent public functionality."""
 
+import contextlib
 from importlib import metadata
 
 from . import _file_transfer as fs  # nopycln: import
@@ -48,9 +49,11 @@
     lattice,
 )
 from ._workflow.electron import wait  # nopycln: import
-from ._workflow.qelectron import qelectron  # nopycln: import
 from .executor.utils import get_context  # nopycln: import
-from .quantum import QCluster  # nopycln: import
+
+with contextlib.suppress(ImportError):
+    from ._workflow.qelectron import qelectron  # nopycln: import
+    from .quantum import QCluster  # nopycln: import
 
 __all__ = [s for s in dir() if not s.startswith("_")]
 

covalent/_shared_files/qelectron_utils.py

@@ -14,10 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import importlib
+import inspect
+from typing import Any, Tuple
 
-from covalent.quantum.qserver.database import Database
+import cloudpickle
+from pennylane._device import Device
 
 from .logger import app_log
+from .pickling import _qml_mods_pickle
+
+_IMPORT_PATH_SEPARATOR = ":"
 
 
 def get_qelectron_db_path(dispatch_id: str, task_id: int):
@@ -28,6 +35,8 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int):
     AS WHERE THE USER'S TASK FUNCTION IS BEING RUN.
     """
 
+    from covalent.quantum.qserver.database import Database
+
     database = Database()
 
     db_path = database.get_db_path(dispatch_id=dispatch_id, node_id=task_id)
@@ -38,3 +47,50 @@ def get_qelectron_db_path(dispatch_id: str, task_id: int):
     else:
         app_log.debug(f"Qelectron database not found for task {task_id}")
         return None
+
+
+@_qml_mods_pickle
+def cloudpickle_serialize(obj):
+    return cloudpickle.dumps(obj)
+
+
+def cloudpickle_deserialize(obj):
+    return cloudpickle.loads(obj)
+
+
+def select_first_executor(qnode, executors):
+    """Selects the first executor to run the qnode"""
+    return executors[0]
+
+
+def get_import_path(obj) -> Tuple[str, str]:
+    """
+    Determine the import path of an object.
+    """
+    if module := inspect.getmodule(obj):
+        module_path = module.__name__
+        class_name = obj.__name__
+        return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}"
+    raise RuntimeError(f"Unable to determine import path for {obj}.")
+
+
+def import_from_path(path: str) -> Any:
+    """
+    Import a class from a path.
+    """
+    module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR)
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)
+
+
+def get_original_shots(dev: Device):
+    """
+    Recreate vector of shots if device has a shot vector.
+    """
+    if not dev.shot_vector:
+        return dev.shots
+
+    shot_sequence = []
+    for shots in dev.shot_vector:
+        shot_sequence.extend([shots.shots] * shots.copies)
+    return type(dev.shot_vector)(shot_sequence)

covalent/_shared_files/qresult_utils.py

@@ -23,7 +23,7 @@
 from pennylane.tape import QuantumTape
 
 from .._workflow.qdevice import QEDevice
-from .utils import get_original_shots
+from .qelectron_utils import get_original_shots
 
 
 def re_execute(

covalent/_shared_files/utils.py

@@ -16,19 +16,14 @@
 
 """General utils for Covalent."""
 
-import importlib
 import inspect
 import shutil
 import socket
 from datetime import timedelta
-from typing import Any, Callable, Dict, List, Tuple
-
-import cloudpickle
-from pennylane._device import Device
+from typing import Callable, Dict, List, Tuple
 
 from . import logger
 from .config import get_config
-from .pickling import _qml_mods_pickle
 
 app_log = logger.app_log
 log_stack_info = logger.log_stack_info
@@ -37,9 +32,6 @@
 DEFAULT_UI_PORT = get_config("user_interface.port")
 
 
-_IMPORT_PATH_SEPARATOR = ":"
-
-
 def get_ui_url(path):
     baseUrl = f"http://{DEFAULT_UI_ADDRESS}:{DEFAULT_UI_PORT}"
     return f"{baseUrl}{path}"
@@ -264,49 +256,16 @@ def copy_file_locally(src_uri, dest_uri):
     shutil.copyfile(src_path, dest_path)
 
 
-@_qml_mods_pickle
-def cloudpickle_serialize(obj):
-    return cloudpickle.dumps(obj)
-
-
-def cloudpickle_deserialize(obj):
-    return cloudpickle.loads(obj)
-
-
-def select_first_executor(qnode, executors):
-    """Selects the first executor to run the qnode"""
-    return executors[0]
-
-
-def get_import_path(obj) -> Tuple[str, str]:
-    """
-    Determine the import path of an object.
+def get_qelectron_db_path(dispatch_id: str, task_id: int):
     """
-    module = inspect.getmodule(obj)
-    if module:
-        module_path = module.__name__
-        class_name = obj.__name__
-        return f"{module_path}{_IMPORT_PATH_SEPARATOR}{class_name}"
-    raise RuntimeError(f"Unable to determine import path for {obj}.")
+    Return the path to the Qelectron database for a given dispatch_id and task_id.
 
-
-def import_from_path(path: str) -> Any:
-    """
-    Import a class from a path.
+    This is a proxy to qelectron_utils.get_qelectron_db_path() for removing qelectron dependency.
     """
-    module_path, class_name = path.split(_IMPORT_PATH_SEPARATOR)
-    module = importlib.import_module(module_path)
-    return getattr(module, class_name)
 
+    try:
+        from .qelectron_utils import get_qelectron_db_path
 
-def get_original_shots(dev: Device):
-    """
-    Recreate vector of shots if device has a shot vector.
-    """
-    if not dev.shot_vector:
-        return dev.shots
-
-    shot_sequence = []
-    for shots in dev.shot_vector:
-        shot_sequence.extend([shots.shots] * shots.copies)
-    return type(dev.shot_vector)(shot_sequence)
+        return get_qelectron_db_path(dispatch_id, task_id)
+    except ImportError:
+        return None

covalent/_workflow/qelectron.py

@@ -19,7 +19,7 @@
 
 import pennylane as qml
 
-from .._shared_files.utils import get_import_path, get_original_shots
+from .._shared_files.qelectron_utils import get_import_path, get_original_shots
 from ..quantum.qcluster import QCluster
 from ..quantum.qcluster.base import AsyncBaseQCluster, BaseQExecutor
 from ..quantum.qcluster.simulator import Simulator

covalent/_workflow/qnode.py

@@ -25,9 +25,9 @@
 
 from .._results_manager.qresult import QNodeFutureResult
 from .._shared_files import logger
+from .._shared_files.qelectron_utils import get_original_shots
 from .._shared_files.qinfo import QElectronInfo, QNodeSpecs
 from .._shared_files.qresult_utils import re_execute
-from .._shared_files.utils import get_original_shots
 from ..executor.qbase import BaseQExecutor
 from .qdevice import QEDevice
 

covalent/executor/__init__.py

@@ -31,7 +31,6 @@
 
 from .._shared_files import logger
 from .._shared_files.config import get_config, update_config
-from ..quantum import QCluster, Simulator
 from .base import BaseExecutor
 
 app_log = logger.app_log
@@ -284,6 +283,8 @@ class _QExecutorManager:
     """
 
     def __init__(self):
+        from ..quantum import QCluster, Simulator
+
         # Dictionary mapping executor name to executor class
         self.executor_plugins_map: Dict[str, Any] = {
             "QCluster": QCluster,
@@ -370,11 +371,12 @@ def validate_module(self, module_obj) -> None:
 
 
 _executor_manager = _ExecutorManager()
-_qexecutor_manager = _QExecutorManager()
-
 for name in _executor_manager.executor_plugins_map:
     plugin_class = _executor_manager.executor_plugins_map[name]
     globals()[plugin_class.__name__] = plugin_class
 
-for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values():
-    globals()[qexecutor_cls.__name__] = qexecutor_cls
+# Only creating the qexecutor manager if its requirements are installed
+with contextlib.suppress(ImportError):
+    _qexecutor_manager = _QExecutorManager()
+    for qexecutor_cls in _qexecutor_manager.executor_plugins_map.values():
+        globals()[qexecutor_cls.__name__] = qexecutor_cls