Update 223P to public advisory review version

.github/workflows/ci.yml

@@ -3,6 +3,10 @@ name: continuous integration
 on:
   push
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   styling:
@@ -59,13 +63,13 @@ jobs:
       - name: unit tests
         run: poetry run pytest tests/unit --cov=./ --cov-report=xml
       - name: integration tests
-        run: poetry run pytest tests/integration
+        run: poetry run pytest tests/integration -k 'topquadrant'
       - name: bacnet tests
         run: |
           cd tests/integration/fixtures/bacnet
           docker compose build device buildingmotif
           docker compose run -d device
-          docker compose run buildingmotif poetry run pytest -m bacnet
+          docker compose run buildingmotif poetry run pytest -m bacnet --skip-library-tests
           docker compose down
       - name: build tests
         run: poetry build

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
         entry: poetry run flake8 buildingmotif
 # can't poetry run becuase not present in repository https://github.com/pre-commit/mirrors-mypy
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
     -   id: mypy
         args: ["--install-types", "--non-interactive", "--ignore-missing-imports", "--follow-imports=skip", "--disable-error-code=import-untyped"]

buildingmotif/api/app.py

@@ -1,4 +1,5 @@
 import os
+from typing import Optional
 
 from flask import Flask, current_app
 from flask_api import status
@@ -42,7 +43,7 @@ def _after_error(error):
     return str(error), status.HTTP_500_INTERNAL_SERVER_ERROR
 
 
-def create_app(DB_URI):
+def create_app(DB_URI, shacl_engine: Optional[str] = "pyshacl"):
     """Creates a Flask API.
 
     :param db_uri: database URI
@@ -54,7 +55,7 @@ def create_app(DB_URI):
     app.config.from_mapping(
         DB_URI=DB_URI,
     )
-    app.building_motif = BuildingMOTIF(app.config["DB_URI"])
+    app.building_motif = BuildingMOTIF(app.config["DB_URI"], shacl_engine=shacl_engine)
 
     app.after_request(_after_request)
     app.register_error_handler(Exception, _after_error)

buildingmotif/dataclasses/library.py

@@ -20,6 +20,7 @@
 from buildingmotif.schemas import validate_libraries_yaml
 from buildingmotif.template_compilation import compile_template_spec
 from buildingmotif.utils import (
+    copy_graph,
     get_ontology_files,
     get_template_parts_from_shape,
     shacl_inference,
@@ -141,6 +142,8 @@ def load(
         directory: Optional[str] = None,
         name: Optional[str] = None,
         overwrite: Optional[bool] = True,
+        infer_templates: Optional[bool] = True,
+        run_shacl_inference: Optional[bool] = True,
     ) -> "Library":
         """Loads a library from the database or an external source.
         When specifying a path to load a library or ontology_graph from,
@@ -162,6 +165,12 @@ def load(
         :param overwrite: if true, replace any existing copy of the
             library, defaults to True
         :type overwrite: Optional[true], optional
+        :param infer_templates: if true, infer shapes from the ontology graph,
+            defaults to True
+        :type infer_templates: Optional[bool], optional
+        :param run_shacl_inference: if true, run SHACL inference on the ontology graph,
+            using the BuildingMOTIF SHACL engine, defaults to True
+        :type run_shacl_inference: Optional[bool], optional
         :return: the loaded library
         :rtype: Library
         :raises Exception: if the library cannot be loaded
@@ -180,7 +189,12 @@ def load(
                 ontology_graph.parse(
                     ontology_graph_path, format=guess_format(ontology_graph_path)
                 )
-            return cls._load_from_ontology(ontology_graph, overwrite=overwrite)
+            return cls._load_from_ontology(
+                ontology_graph,
+                overwrite=overwrite,
+                infer_templates=infer_templates,
+                run_shacl_inference=run_shacl_inference,
+            )
         elif directory is not None:
             if resource_exists("buildingmotif.libraries", directory):
                 logging.debug(f"Loading builtin library: {directory}")
@@ -191,7 +205,12 @@ def load(
                 src = pathlib.Path(directory)
             if not src.exists():
                 raise Exception(f"Directory {src} does not exist")
-            return cls._load_from_directory(src, overwrite=overwrite)
+            return cls._load_from_directory(
+                src,
+                overwrite=overwrite,
+                infer_templates=infer_templates,
+                run_shacl_inference=run_shacl_inference,
+            )
         elif name is not None:
             bm = get_building_motif()
             db_library = bm.table_connection.get_db_library_by_name(name)
@@ -215,7 +234,11 @@ def _load_from_db(cls, id: int) -> "Library":
 
     @classmethod
     def _load_from_ontology(
-        cls, ontology: rdflib.Graph, overwrite: Optional[bool] = True
+        cls,
+        ontology: rdflib.Graph,
+        overwrite: Optional[bool] = True,
+        infer_templates: Optional[bool] = True,
+        run_shacl_inference: Optional[bool] = True,
     ) -> "Library":
         """
         Load a library from an ontology graph. This proceeds as follows.
@@ -229,14 +252,18 @@ def _load_from_ontology(
         :type ontology: rdflib.Graph
         :param overwrite: if true, overwrite the existing copy of the Library
         :type overwrite: bool
+        :param infer_templates: if true, infer shapes from the ontology graph
+        :type infer_templates: bool
+        :param run_shacl_inference: if true, run SHACL inference on the ontology graph
+        :type run_shacl_inference: bool
         :return: the loaded Library
         :rtype: "Library"
         """
         # get the name of the ontology; this will be the name of the library
         # any=False will raise an error if there is more than one ontology defined  in the graph
         ontology_name = ontology.value(
             predicate=rdflib.RDF.type, object=rdflib.OWL.Ontology, any=False
-        )
+        ) or rdflib.URIRef("urn:unnamed/")
 
         if not overwrite:
             if cls._library_exists(ontology_name):
@@ -248,12 +275,16 @@ def _load_from_ontology(
         # expand the ontology graph before we insert it into the database. This will ensure
         # that the output of compiled models will not contain triples that really belong to
         # the ontology
-        ontology = shacl_inference(ontology, engine=get_building_motif().shacl_engine)
+        if run_shacl_inference:
+            ontology = shacl_inference(
+                ontology, engine=get_building_motif().shacl_engine
+            )
 
         lib = cls.create(ontology_name, overwrite=overwrite)
 
-        # infer shapes from any class/nodeshape candidates in the graph
-        lib._infer_shapes_from_graph(ontology)
+        if infer_templates:
+            # infer shapes from any class/nodeshape candidates in the graph
+            lib._infer_templates_from_graph(ontology)
 
         # load the ontology graph as a shape_collection
         shape_col_id = lib.get_shape_collection().id
@@ -263,12 +294,26 @@ def _load_from_ontology(
 
         return lib
 
-    def _infer_shapes_from_graph(self, graph: rdflib.Graph):
-        """Infer shapes from a graph and add them to this library.
+    def _infer_templates_from_graph(self, graph: rdflib.Graph):
+        """Infer templates from a graph (by interpreting shapes) and add them to this library.
 
-        :param graph: graph to infer shapes from
+        :param graph: graph to infer templates from
         :type graph: rdflib.Graph
         """
+        # add all imports to the same graph so we can resolve everything
+        imports_closure = copy_graph(graph)
+        # import dependencies into 'graph'
+        # get all imports from the graph
+        for dependency in graph.objects(predicate=rdflib.OWL.imports):
+            # attempt to load from BuildingMOTIF
+            try:
+                lib = Library.load(name=str(dependency))
+                imports_closure += lib.get_shape_collection().graph
+            except Exception as e:  # TODO: replace with a more specific exception
+                logging.warning(
+                    f"An ontology could not resolve a dependency on {dependency} ({e}). Check this is loaded into BuildingMOTIF"
+                )
+                continue
         class_candidates = set(graph.subjects(rdflib.RDF.type, rdflib.OWL.Class))
         shape_candidates = set(graph.subjects(rdflib.RDF.type, rdflib.SH.NodeShape))
         candidates = class_candidates.intersection(shape_candidates)
@@ -277,19 +322,30 @@ def _infer_shapes_from_graph(self, graph: rdflib.Graph):
         for candidate in candidates:
             assert isinstance(candidate, rdflib.URIRef)
             # TODO: mincount 0 (or unspecified) should be optional args on the generated template
-            partial_body, deps = get_template_parts_from_shape(candidate, graph)
+            partial_body, deps = get_template_parts_from_shape(
+                candidate, imports_closure
+            )
             templ = self.create_template(str(candidate), partial_body)
             dependency_cache[templ.id] = deps
             template_id_lookup[str(candidate)] = templ.id
 
         self._resolve_template_dependencies(template_id_lookup, dependency_cache)
 
-    def _load_shapes_from_directory(self, directory: pathlib.Path):
+    def _load_shapes_from_directory(
+        self,
+        directory: pathlib.Path,
+        infer_templates: Optional[bool] = True,
+        run_shacl_inference: Optional[bool] = True,
+    ):
         """Helper method to read all graphs in the given directory into this
         library.
 
         :param directory: directory containing graph files
         :type directory: pathlib.Path
+        :param infer_templates: if true, infer shapes from the ontology graph
+        :type infer_templates: bool
+        :param run_shacl_inference: if true, run SHACL inference on the ontology graph
+        :type run_shacl_inference: bool
         """
         shape_col_id = self.get_shape_collection().id
         assert shape_col_id is not None  # this should always pass
@@ -302,12 +358,21 @@ def _load_shapes_from_directory(self, directory: pathlib.Path):
                     f"Could not parse file {filename}: {e}"
                 )
                 raise e
+        if run_shacl_inference:
+            shape_col.graph = shacl_inference(
+                shape_col.graph, engine=get_building_motif().shacl_engine
+            )
         # infer shapes from any class/nodeshape candidates in the graph
-        self._infer_shapes_from_graph(shape_col.graph)
+        if infer_templates:
+            self._infer_templates_from_graph(shape_col.graph)
 
     @classmethod
     def _load_from_directory(
-        cls, directory: pathlib.Path, overwrite: Optional[bool] = True
+        cls,
+        directory: pathlib.Path,
+        overwrite: Optional[bool] = True,
+        infer_templates: Optional[bool] = True,
+        run_shacl_inference: Optional[bool] = True,
     ) -> "Library":
         """
         Load a library from a directory.
@@ -319,6 +384,10 @@ def _load_from_directory(
         :type directory: pathlib.Path
         :param overwrite: if true, overwrite the existing copy of the Library
         :type overwrite: bool
+        :param infer_templates: if true, infer shapes from the ontology graph
+        :type infer_templates: bool
+        :param run_shacl_inference: if true, run SHACL inference on the ontology graph
+        :type run_shacl_inference: bool
         :raises e: if cannot create template
         :raises e: if cannot resolve dependencies
         :return: library
@@ -416,6 +485,7 @@ def _resolve_dependency(
         # check documentation for skip_uri for what URIs get skipped
         if skip_uri(dep["template"]):
             return
+
         # if the dependency is not in the local cache, then search through this library's imports
         # for the template
         for imp in self.graph_imports:

buildingmotif/dataclasses/model.py

@@ -191,6 +191,9 @@ def validate(
         # TODO: do we want to preserve the materialized triples added to data_graph via reasoning?
         data_graph = copy_graph(self.graph)
 
+        # remove imports from data graph
+        data_graph.remove((None, OWL.imports, None))
+
         # validate the data graph
         valid, report_g, report_str = shacl_validate(
             data_graph, shapeg, engine=self._bm.shacl_engine
@@ -276,7 +279,7 @@ def test_model_against_shapes(
             ontology_graph = ontology_graph.skolemize()
 
             valid, report_g, report_str = shacl_validate(
-                temp_model_graph, ontology_graph
+                temp_model_graph, ontology_graph, engine=self._bm.shacl_engine
             )
 
             results[shape_uri] = ValidationContext(

buildingmotif/dataclasses/template.py

@@ -19,6 +19,7 @@
 from buildingmotif.template_matcher import Mapping, TemplateMatcher
 from buildingmotif.utils import (
     PARAM,
+    _strip_param,
     combine_graphs,
     copy_graph,
     remove_triples_with_node,
@@ -280,17 +281,18 @@ def inline_dependencies(self) -> "Template":
             # replace dependency parameters with the names they inherit
             # through the provided bindings
             rename_params: Dict[str, str] = {
-                ours: theirs for ours, theirs in dep.args.items()
+                ours: _strip_param(theirs) for ours, theirs in dep.args.items()
             }
             # replace all parameters *not* mentioned in the args by prefixing
             # them with the 'name' parameter binding; this is guaranteed
             # to exist
-            name_prefix = dep.args.get("name")
+            name_prefix = _strip_param(dep.args["name"])
             # for each parameter in the dependency...
             for param in deptempl.parameters:
                 # if it does *not* have a mapping in the dependency, then
                 # prefix the parameter with the value of the 'name' binding
                 # to scope it properly
+                param = _strip_param(param)
                 if param not in dep.args and param != "name":
                     rename_params[param] = f"{name_prefix}-{param}"
 

buildingmotif/namespaces.py

@@ -31,6 +31,7 @@
 
 BM = Namespace("https://nrel.gov/BuildingMOTIF#")
 CONSTRAINT = Namespace("https://nrel.gov/BuildingMOTIF/constraints#")
+S223 = Namespace("http://data.ashrae.org/standard223#")
 
 A = RDF.type
 

buildingmotif/utils.py

@@ -5,7 +5,7 @@
 from dataclasses import dataclass
 from itertools import chain
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
 
 import pyshacl  # type: ignore
 from rdflib import BNode, Graph, Literal, URIRef
@@ -22,6 +22,16 @@
 _gensym_counter = 0
 
 
+def _strip_param(param: Union[Node, str]) -> str:
+    """
+    Strips all PARAM namespaces from the input parameter
+    """
+    param = str(param)
+    while param.startswith(PARAM):
+        param = param[len(PARAM) :]
+    return param
+
+
 def _gensym(prefix: str = "p") -> URIRef:
     """
     Generate a unique identifier.
@@ -208,7 +218,9 @@ def get_template_parts_from_shape(
     for pshape in pshapes:
         property_path = shape_graph.value(pshape, SH["path"])
         if property_path is None:
-            raise Exception(f"no sh:path detected on {shape_name}")
+            raise Exception(
+                f"no sh:path detected on {shape_name} property shape {pshape}"
+            )
         # TODO: expand otypes to include sh:in, sh:or, or no datatype at all!
         otypes = list(
             shape_graph.objects(
@@ -576,6 +588,7 @@ def shacl_validate(
             )
             pass
 
+    data_graph = data_graph + (shape_graph or Graph())
     return pyshacl.validate(
         data_graph,
         shacl_graph=shape_graph,