From ce8da083d0bbeeec1de1e3e73ad85defe98448d3 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Fri, 16 Sep 2022 11:16:34 -0400 Subject: [PATCH 01/29] Filter SHACL statements from data graph's ontology mix-in References: * https://github.com/RDFLib/pySHACL/issues/89 Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 77e2f76..5a033eb 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -142,6 +142,37 @@ def main() -> None: _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) ontology_graph.parse(arg_ontology_graph) + # Filter the graph pyshacl uses as its ontology mix-in to exclude + # all SHACL-related triples. + # This is done because, at the time of pyshacl 0.20.0, the entirety + # of the ontology graph is mixed into the data graph. UCO 1.0.0 + # includes some mechanisms to cross-check SHACL PropertyShapes + # versus OWL property definitions. Because of the mix-in, all of + # the ontology graph (.validate ont_graph kwarg) is reviewed by the + # SHACL graph (.validate shacl_graph kwarg), so for UCO 1.0.0 that + # adds around 30 seconds to each case_validate call, redundantly + # reviewing UCO. + # The ontology graph (.validate ont_graph kwarg) is currently + # believed to never need to know about SHACL concepts. + ontology_graph_without_shacl = rdflib.Graph() + SH_prefix = str(rdflib.SH) + for triple in ontology_graph.triples((None, None, None)): + skip_triple = False + for triple_part in triple: + if isinstance(triple_part, rdflib.URIRef): + if str(triple_part).startswith(SH_prefix): + skip_triple = True + if skip_triple: + break + if skip_triple: + continue + ontology_graph_without_shacl.add(triple) + # _logger.debug("len(ontology_graph) = %d.", len(ontology_graph)) + # _logger.debug("len(ontology_graph_without_shacl) = %d.", len(ontology_graph_without_shacl)) + # At the time of CASE 1.0.0, this was the debug output: + # DEBUG:__init__.py:len(ontology_graph) = 13499. + # DEBUG:__init__.py:len(ontology_graph_without_shacl) = 7639. + # Determine output format. # pySHACL's determination of output formatting is handled solely # through the -f flag. Other CASE CLI tools handle format @@ -158,7 +189,7 @@ def main() -> None: validate_result = pyshacl.validate( data_graph, shacl_graph=ontology_graph, - ont_graph=ontology_graph, + ont_graph=ontology_graph_without_shacl, inference=args.inference, abort_on_first=args.abort, allow_warnings=True if args.allow_warnings else False, From 866345246e3ea5748f327b96d750dd3ed9d34e96 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Fri, 12 May 2023 09:27:20 -0400 Subject: [PATCH 02/29] Adjust documentation of exposed API I stumbled across this being a potential point of confusion in the Exifread mapping repository's PR 21. References: * https://github.com/casework/CASE-Implementation-PyPI-Exifread/pull/21 Signed-off-by: Alex Nelson --- case_utils/local_uuid.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/case_utils/local_uuid.py b/case_utils/local_uuid.py index 85801b3..881c577 100644 --- a/case_utils/local_uuid.py +++ b/case_utils/local_uuid.py @@ -13,10 +13,14 @@ """ This library is a wrapper for uuid, provided to generate repeatable UUIDs if requested. + +The function local_uuid() should be used in code where a user could be expected to opt in to non-random UUIDs. """ __version__ = "0.3.2" +__all__ = ["configure", "local_uuid"] + import logging import os import pathlib @@ -34,13 +38,13 @@ def configure() -> None: """ - This function is part of setting up demo_uuid() to generate non-random UUIDs. See demo_uuid() documentation for further setup notes. + This function is part of setting up _demo_uuid() to generate non-random UUIDs. See _demo_uuid() documentation for further setup notes. """ global DEMO_UUID_BASE if os.getenv("DEMO_UUID_REQUESTING_NONRANDOM") == "NONRANDOM_REQUESTED": warnings.warn( - "Environment variable DEMO_UUID_REQUESTING_NONRANDOM is deprecated. See case_utils.local_uuid.demo_uuid for usage notes on its replacement, CASE_DEMO_NONRANDOM_UUID_BASE. Proceeding with random UUIDs.", + "Environment variable DEMO_UUID_REQUESTING_NONRANDOM is deprecated. See case_utils.local_uuid._demo_uuid for usage notes on its replacement, CASE_DEMO_NONRANDOM_UUID_BASE. Proceeding with random UUIDs.", FutureWarning, ) return @@ -109,10 +113,12 @@ def configure() -> None: DEMO_UUID_BASE = "/".join(demo_uuid_base_parts) -def demo_uuid() -> str: +def _demo_uuid() -> str: """ This function generates a repeatable UUID, drawing on non-varying elements of the environment and process call for entropy. + This function is not intended to be called outside of this module. Instead, local_uuid() should be called. + WARNING: This function was developed for use ONLY for reducing (but not eliminating) version-control edits to identifiers when generating sample data. It creates UUIDs that are decidedly NOT random, and should remain consistent on repeated calls to the importing script. To prevent accidental non-random UUID usage, two setup steps need to be done before calling this function: @@ -148,4 +154,4 @@ def local_uuid() -> str: if DEMO_UUID_BASE is None: return str(uuid.uuid4()) else: - return demo_uuid() + return _demo_uuid() From 9b6a4220b66ad54ed919a41fdac20789fbfbf20e Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Tue, 23 May 2023 08:47:58 -0400 Subject: [PATCH 03/29] Start specifying RDFLib version ceilings This sets `case-utils` and downstream Python tooling to take care in reviewing new RDFLib releases for breaking changes. RDFLib Issue 2402 and Discussion 2395 outline an upcoming development period for RDFLib that may see more frequent SEMVER-major version releases. References: * https://github.com/RDFLib/rdflib/pull/2402 Signed-off-by: Alex Nelson --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e5ffda7..7420867 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,7 @@ include_package_data = true install_requires = pandas pyshacl - rdflib >= 6.2.0 + rdflib >= 6.2.0, < 7.0.0 requests tabulate packages = find: From 2df147a86c65ec455daec555b9ea4f397a617b14 Mon Sep 17 00:00:00 2001 From: kchason Date: Mon, 24 Jul 2023 20:36:21 -0400 Subject: [PATCH 04/29] Add validate() function for programmatic access --- case_utils/case_validate/__init__.py | 246 ++++++++++++++++++--------- 1 file changed, 167 insertions(+), 79 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 5ef2952..1b0a249 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -36,11 +36,12 @@ import logging import os import sys -import typing import warnings +from typing import Optional, Set, Union, Dict, Tuple import pyshacl # type: ignore import rdflib +from rdflib import Graph import case_utils.ontology from case_utils.ontology.version_info import ( @@ -64,13 +65,168 @@ class NonExistentCDOConceptWarning(UserWarning): pass +class ValidationResult: + conforms: bool + graph: Graph + text: str + undefined_concepts: Set[rdflib.URIRef] + + def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: + """ + Determine if a concept is part of the CDO ontology. + + :param n_concept: The concept to check. + :return: whether the concept is part of the CDO ontology. + """ concept_iri = str(n_concept) return concept_iri.startswith( "https://ontology.unifiedcyberontology.org/" ) or concept_iri.startswith("https://ontology.caseontology.org/") +def get_ontology_graph(case_version: Optional[str] = None, + supplemental_graphs: Optional[list[str]] = None) -> rdflib.Graph: + """ + Get the ontology graph for the given case_version and any supplemental graphs. + + :param case_version: the version of the CASE ontology to use. If None, the most recent version will be used. + :param supplemental_graphs: a list of supplemental graphs to use. If None, no supplemental graphs will be used. + :return: the ontology graph against which to validate the data graph. + """ + ontology_graph = rdflib.Graph() + if case_version and case_version != "none": + ttl_filename = case_version + ".ttl" + _logger.debug("ttl_filename = %r.", ttl_filename) + ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) + ontology_graph.parse(data=ttl_data, format="turtle") + if supplemental_graphs: + for arg_ontology_graph in supplemental_graphs: + _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) + ontology_graph.parse(arg_ontology_graph) + return ontology_graph + + +def get_invalid_cdo_concepts(data_graph: Graph, ontology_graph: Graph) -> Set[rdflib.URIRef]: + """ + Get the set of concepts in the data graph that are not part of the CDO ontology. + + :param data_graph: The data graph to validate. + :param ontology_graph: The ontology graph to use for validation. + :return: The list of concepts in the data graph that are not part of the CDO ontology. + """ + # Construct set of CDO concepts for data graph concept-existence review. + cdo_concepts: Set[rdflib.URIRef] = set() + + for n_structural_class in [ + NS_OWL.Class, + NS_OWL.AnnotationProperty, + NS_OWL.DatatypeProperty, + NS_OWL.ObjectProperty, + NS_RDFS.Datatype, + NS_SH.NodeShape, + NS_SH.PropertyShape, + NS_SH.Shape, + ]: + for ontology_triple in ontology_graph.triples( + (None, NS_RDF.type, n_structural_class) + ): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + if concept_is_cdo_concept(ontology_triple[0]): + cdo_concepts.add(ontology_triple[0]) + for n_ontology_predicate in [ + NS_OWL.backwardCompatibleWith, + NS_OWL.imports, + NS_OWL.incompatibleWith, + NS_OWL.priorVersion, + NS_OWL.versionIRI, + ]: + for ontology_triple in ontology_graph.triples( + (None, n_ontology_predicate, None) + ): + assert isinstance(ontology_triple[0], rdflib.URIRef) + assert isinstance(ontology_triple[2], rdflib.URIRef) + cdo_concepts.add(ontology_triple[0]) + cdo_concepts.add(ontology_triple[2]) + for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + cdo_concepts.add(ontology_triple[0]) + + # Also load historical ontology and version IRIs. + ontology_and_version_iris_data = importlib.resources.read_text( + case_utils.ontology, "ontology_and_version_iris.txt" + ) + for line in ontology_and_version_iris_data.split("\n"): + cleaned_line = line.strip() + if cleaned_line == "": + continue + cdo_concepts.add(rdflib.URIRef(cleaned_line)) + + data_cdo_concepts: Set[rdflib.URIRef] = set() + for data_triple in data_graph.triples((None, None, None)): + for data_triple_member in data_triple: + if isinstance(data_triple_member, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member): + data_cdo_concepts.add(data_triple_member) + elif isinstance(data_triple_member, rdflib.Literal): + if isinstance(data_triple_member.datatype, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member.datatype): + data_cdo_concepts.add(data_triple_member.datatype) + + return data_cdo_concepts - cdo_concepts + + +def validate(input_file: str, case_version: Optional[str] = None, supplemental_graphs: Optional[list[str]] = None, + abort_on_first: bool = False) -> ValidationResult: + """ + Validate the given data graph against the given CASE ontology version and supplemental graphs. + + :param input_file: The path to the file containing the data graph to validate. + :param case_version: The version of the CASE ontology to use. If None, the most recent version will be used. + :param supplemental_graphs: The supplemental graphs to use. If None, no supplemental graphs will be used. + :param abort_on_first: Whether to abort on the first validation error. + :return: The validation result object containing the defined properties. + """ + # Convert the data graph string to a rdflib.Graph object. + data_graph = rdflib.Graph() + data_graph.parse(input_file) + + # Get the ontology graph from the case_version and supplemental_graphs arguments + ontology_graph: Graph = get_ontology_graph(case_version, supplemental_graphs) + + # Get the undefined CDO concepts + undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) + + # Validate data graph against ontology graph. + validate_result: Tuple[ + bool, Union[Exception, bytes, str, rdflib.Graph], str + ] = pyshacl.validate( + data_graph, + shacl_graph=ontology_graph, + ont_graph=ontology_graph, + inference="none", + meta_shacl=False, + abort_on_first=abort_on_first, + allow_infos=False, + allow_warnings=False, + debug=False, + do_owl_imports=False, + ) + + # Relieve RAM of the data graph after validation has run. + del data_graph + + result = ValidationResult() + result.conforms = validate_result[0] + result.graph = validate_result[1] + result.text = validate_result[2] + result.undefined_cdo_concepts = undefined_cdo_concepts + + return result + + def main() -> None: parser = argparse.ArgumentParser( description="CASE wrapper to pySHACL command line tool." @@ -170,83 +326,16 @@ def main() -> None: _logger.debug("in_graph = %r.", in_graph) data_graph.parse(in_graph) - ontology_graph = rdflib.Graph() - if args.built_version != "none": - ttl_filename = args.built_version + ".ttl" - _logger.debug("ttl_filename = %r.", ttl_filename) - ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) - ontology_graph.parse(data=ttl_data, format="turtle") - if args.ontology_graph: - for arg_ontology_graph in args.ontology_graph: - _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) - ontology_graph.parse(arg_ontology_graph) - - # Construct set of CDO concepts for data graph concept-existence review. - cdo_concepts: typing.Set[rdflib.URIRef] = set() - - for n_structural_class in [ - NS_OWL.Class, - NS_OWL.AnnotationProperty, - NS_OWL.DatatypeProperty, - NS_OWL.ObjectProperty, - NS_RDFS.Datatype, - NS_SH.NodeShape, - NS_SH.PropertyShape, - NS_SH.Shape, - ]: - for ontology_triple in ontology_graph.triples( - (None, NS_RDF.type, n_structural_class) - ): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - if concept_is_cdo_concept(ontology_triple[0]): - cdo_concepts.add(ontology_triple[0]) - for n_ontology_predicate in [ - NS_OWL.backwardCompatibleWith, - NS_OWL.imports, - NS_OWL.incompatibleWith, - NS_OWL.priorVersion, - NS_OWL.versionIRI, - ]: - for ontology_triple in ontology_graph.triples( - (None, n_ontology_predicate, None) - ): - assert isinstance(ontology_triple[0], rdflib.URIRef) - assert isinstance(ontology_triple[2], rdflib.URIRef) - cdo_concepts.add(ontology_triple[0]) - cdo_concepts.add(ontology_triple[2]) - for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - cdo_concepts.add(ontology_triple[0]) - - # Also load historical ontology and version IRIs. - ontology_and_version_iris_data = importlib.resources.read_text( - case_utils.ontology, "ontology_and_version_iris.txt" - ) - for line in ontology_and_version_iris_data.split("\n"): - cleaned_line = line.strip() - if cleaned_line == "": - continue - cdo_concepts.add(rdflib.URIRef(cleaned_line)) - - data_cdo_concepts: typing.Set[rdflib.URIRef] = set() - for data_triple in data_graph.triples((None, None, None)): - for data_triple_member in data_triple: - if isinstance(data_triple_member, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member): - data_cdo_concepts.add(data_triple_member) - elif isinstance(data_triple_member, rdflib.Literal): - if isinstance(data_triple_member.datatype, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member.datatype): - data_cdo_concepts.add(data_triple_member.datatype) + # Get the ontology graph based on the CASE version and supplemental graphs specified by the CLI + ontology_graph = get_ontology_graph(case_version=args.built_version, supplemental_graphs=args.ontology_graph) - undefined_cdo_concepts = data_cdo_concepts - cdo_concepts + # Get the list of undefined CDO concepts in the graph + undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) for undefined_cdo_concept in sorted(undefined_cdo_concepts): warnings.warn(undefined_cdo_concept, NonExistentCDOConceptWarning) undefined_cdo_concepts_message = ( - "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." - % len(undefined_cdo_concepts) + "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." + % len(undefined_cdo_concepts) ) # Determine output format. @@ -255,14 +344,13 @@ def main() -> None: # determination by output file extension. case_validate will defer # to pySHACL behavior, as other CASE tools don't (at the time of # this writing) have the value "human" as an output format. - validator_kwargs: typing.Dict[str, str] = dict() + validator_kwargs: Dict[str, str] = dict() if args.format != "human": validator_kwargs["serialize_report_graph"] = args.format - validate_result: typing.Tuple[ - bool, typing.Union[Exception, bytes, str, rdflib.Graph], str - ] - validate_result = pyshacl.validate( + validate_result: Tuple[ + bool, Union[Exception, bytes, str, rdflib.Graph], str + ] = pyshacl.validate( data_graph, shacl_graph=ontology_graph, ont_graph=ontology_graph, From f485161889d27cd5fbdb0100ad412ba749b7fcc1 Mon Sep 17 00:00:00 2001 From: kchason Date: Mon, 24 Jul 2023 20:40:04 -0400 Subject: [PATCH 05/29] Fix pre-commit formatting --- case_utils/case_validate/__init__.py | 31 ++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 1b0a249..e3e816a 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -37,7 +37,7 @@ import os import sys import warnings -from typing import Optional, Set, Union, Dict, Tuple +from typing import Dict, Optional, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -85,8 +85,9 @@ def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: ) or concept_iri.startswith("https://ontology.caseontology.org/") -def get_ontology_graph(case_version: Optional[str] = None, - supplemental_graphs: Optional[list[str]] = None) -> rdflib.Graph: +def get_ontology_graph( + case_version: Optional[str] = None, supplemental_graphs: Optional[list[str]] = None +) -> rdflib.Graph: """ Get the ontology graph for the given case_version and any supplemental graphs. @@ -107,7 +108,9 @@ def get_ontology_graph(case_version: Optional[str] = None, return ontology_graph -def get_invalid_cdo_concepts(data_graph: Graph, ontology_graph: Graph) -> Set[rdflib.URIRef]: +def get_invalid_cdo_concepts( + data_graph: Graph, ontology_graph: Graph +) -> Set[rdflib.URIRef]: """ Get the set of concepts in the data graph that are not part of the CDO ontology. @@ -129,7 +132,7 @@ def get_invalid_cdo_concepts(data_graph: Graph, ontology_graph: Graph) -> Set[rd NS_SH.Shape, ]: for ontology_triple in ontology_graph.triples( - (None, NS_RDF.type, n_structural_class) + (None, NS_RDF.type, n_structural_class) ): if not isinstance(ontology_triple[0], rdflib.URIRef): continue @@ -143,7 +146,7 @@ def get_invalid_cdo_concepts(data_graph: Graph, ontology_graph: Graph) -> Set[rd NS_OWL.versionIRI, ]: for ontology_triple in ontology_graph.triples( - (None, n_ontology_predicate, None) + (None, n_ontology_predicate, None) ): assert isinstance(ontology_triple[0], rdflib.URIRef) assert isinstance(ontology_triple[2], rdflib.URIRef) @@ -178,8 +181,12 @@ def get_invalid_cdo_concepts(data_graph: Graph, ontology_graph: Graph) -> Set[rd return data_cdo_concepts - cdo_concepts -def validate(input_file: str, case_version: Optional[str] = None, supplemental_graphs: Optional[list[str]] = None, - abort_on_first: bool = False) -> ValidationResult: +def validate( + input_file: str, + case_version: Optional[str] = None, + supplemental_graphs: Optional[list[str]] = None, + abort_on_first: bool = False, +) -> ValidationResult: """ Validate the given data graph against the given CASE ontology version and supplemental graphs. @@ -327,15 +334,17 @@ def main() -> None: data_graph.parse(in_graph) # Get the ontology graph based on the CASE version and supplemental graphs specified by the CLI - ontology_graph = get_ontology_graph(case_version=args.built_version, supplemental_graphs=args.ontology_graph) + ontology_graph = get_ontology_graph( + case_version=args.built_version, supplemental_graphs=args.ontology_graph + ) # Get the list of undefined CDO concepts in the graph undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) for undefined_cdo_concept in sorted(undefined_cdo_concepts): warnings.warn(undefined_cdo_concept, NonExistentCDOConceptWarning) undefined_cdo_concepts_message = ( - "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." - % len(undefined_cdo_concepts) + "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." + % len(undefined_cdo_concepts) ) # Determine output format. From b509dc503913296f47fc399ab70aa87952ac126d Mon Sep 17 00:00:00 2001 From: kchason Date: Mon, 24 Jul 2023 21:23:46 -0400 Subject: [PATCH 06/29] Fix property reference --- case_utils/case_validate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index e3e816a..765801e 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -229,7 +229,7 @@ def validate( result.conforms = validate_result[0] result.graph = validate_result[1] result.text = validate_result[2] - result.undefined_cdo_concepts = undefined_cdo_concepts + result.undefined_concepts = undefined_cdo_concepts return result From 8c40df118dc0fbc2621bc5e9589bd365f59a0d77 Mon Sep 17 00:00:00 2001 From: kchason Date: Mon, 24 Jul 2023 21:51:54 -0400 Subject: [PATCH 07/29] Make type generic to account for multiple return types --- case_utils/case_validate/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 765801e..b5f4402 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -37,7 +37,7 @@ import os import sys import warnings -from typing import Dict, Optional, Set, Tuple, Union +from typing import Any, Dict, Optional, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -67,7 +67,7 @@ class NonExistentCDOConceptWarning(UserWarning): class ValidationResult: conforms: bool - graph: Graph + graph: Any text: str undefined_concepts: Set[rdflib.URIRef] From f6d48e288449a7ccabfa077285a9709503e303cb Mon Sep 17 00:00:00 2001 From: kchason Date: Mon, 24 Jul 2023 22:01:47 -0400 Subject: [PATCH 08/29] Fix List vs list for casting --- case_utils/case_validate/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index b5f4402..815293e 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -37,7 +37,7 @@ import os import sys import warnings -from typing import Any, Dict, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -86,7 +86,7 @@ def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: def get_ontology_graph( - case_version: Optional[str] = None, supplemental_graphs: Optional[list[str]] = None + case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None ) -> rdflib.Graph: """ Get the ontology graph for the given case_version and any supplemental graphs. @@ -184,7 +184,7 @@ def get_invalid_cdo_concepts( def validate( input_file: str, case_version: Optional[str] = None, - supplemental_graphs: Optional[list[str]] = None, + supplemental_graphs: Optional[List[str]] = None, abort_on_first: bool = False, ) -> ValidationResult: """ From 7621f7f85501ffa6ff78fe9973c476c5e9fb04f8 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Tue, 25 Jul 2023 07:42:23 -0400 Subject: [PATCH 09/29] Use typing symbols directly This patch partially implements a small piece of PR 118. References: * https://github.com/casework/CASE-Utilities-Python/pull/118 Requested-by: kchason Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 5ef2952..6deaaa0 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -36,8 +36,8 @@ import logging import os import sys -import typing import warnings +from typing import Dict, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -182,7 +182,7 @@ def main() -> None: ontology_graph.parse(arg_ontology_graph) # Construct set of CDO concepts for data graph concept-existence review. - cdo_concepts: typing.Set[rdflib.URIRef] = set() + cdo_concepts: Set[rdflib.URIRef] = set() for n_structural_class in [ NS_OWL.Class, @@ -230,7 +230,7 @@ def main() -> None: continue cdo_concepts.add(rdflib.URIRef(cleaned_line)) - data_cdo_concepts: typing.Set[rdflib.URIRef] = set() + data_cdo_concepts: Set[rdflib.URIRef] = set() for data_triple in data_graph.triples((None, None, None)): for data_triple_member in data_triple: if isinstance(data_triple_member, rdflib.URIRef): @@ -255,13 +255,11 @@ def main() -> None: # determination by output file extension. case_validate will defer # to pySHACL behavior, as other CASE tools don't (at the time of # this writing) have the value "human" as an output format. - validator_kwargs: typing.Dict[str, str] = dict() + validator_kwargs: Dict[str, str] = dict() if args.format != "human": validator_kwargs["serialize_report_graph"] = args.format - validate_result: typing.Tuple[ - bool, typing.Union[Exception, bytes, str, rdflib.Graph], str - ] + validate_result: Tuple[bool, Union[Exception, bytes, str, rdflib.Graph], str] validate_result = pyshacl.validate( data_graph, shacl_graph=ontology_graph, From 2d3a65b901b118354a8943157cc314d648d2e950 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 25 Jul 2023 20:59:28 -0400 Subject: [PATCH 10/29] Feedback from PR --- case_utils/case_validate/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 815293e..df02047 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -77,7 +77,7 @@ def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: Determine if a concept is part of the CDO ontology. :param n_concept: The concept to check. - :return: whether the concept is part of the CDO ontology. + :return: whether the concept is part of the CDO ontologies. """ concept_iri = str(n_concept) return concept_iri.startswith( @@ -95,12 +95,14 @@ def get_ontology_graph( :param supplemental_graphs: a list of supplemental graphs to use. If None, no supplemental graphs will be used. :return: the ontology graph against which to validate the data graph. """ + if not case_version or case_version == "none": + case_version = CURRENT_CASE_VERSION + ontology_graph = rdflib.Graph() - if case_version and case_version != "none": - ttl_filename = case_version + ".ttl" - _logger.debug("ttl_filename = %r.", ttl_filename) - ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) - ontology_graph.parse(data=ttl_data, format="turtle") + ttl_filename = case_version + ".ttl" + _logger.debug("ttl_filename = %r.", ttl_filename) + ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) + ontology_graph.parse(data=ttl_data, format="turtle") if supplemental_graphs: for arg_ontology_graph in supplemental_graphs: _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) @@ -112,11 +114,11 @@ def get_invalid_cdo_concepts( data_graph: Graph, ontology_graph: Graph ) -> Set[rdflib.URIRef]: """ - Get the set of concepts in the data graph that are not part of the CDO ontology. + Get the set of concepts in the data graph that are not part of the CDO ontologies. :param data_graph: The data graph to validate. :param ontology_graph: The ontology graph to use for validation. - :return: The list of concepts in the data graph that are not part of the CDO ontology. + :return: The set of concepts in the data graph that are not part of the CDO ontologies. """ # Construct set of CDO concepts for data graph concept-existence review. cdo_concepts: Set[rdflib.URIRef] = set() @@ -186,6 +188,7 @@ def validate( case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None, abort_on_first: bool = False, + inference: Optional[str] = "none", ) -> ValidationResult: """ Validate the given data graph against the given CASE ontology version and supplemental graphs. @@ -194,6 +197,7 @@ def validate( :param case_version: The version of the CASE ontology to use. If None, the most recent version will be used. :param supplemental_graphs: The supplemental graphs to use. If None, no supplemental graphs will be used. :param abort_on_first: Whether to abort on the first validation error. + :param inference: The type of inference to use. If "none", no inference will be used. :return: The validation result object containing the defined properties. """ # Convert the data graph string to a rdflib.Graph object. @@ -213,7 +217,7 @@ def validate( data_graph, shacl_graph=ontology_graph, ont_graph=ontology_graph, - inference="none", + inference=inference, meta_shacl=False, abort_on_first=abort_on_first, allow_infos=False, From 86ed417729d2f139c5290aef07a1bf486923e619 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 25 Jul 2023 21:13:11 -0400 Subject: [PATCH 11/29] Instantiate properties as instance variables instead of class --- case_utils/case_validate/__init__.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index df02047..52369a7 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -66,10 +66,17 @@ class NonExistentCDOConceptWarning(UserWarning): class ValidationResult: - conforms: bool - graph: Any - text: str - undefined_concepts: Set[rdflib.URIRef] + def __init__( + self, + conforms: bool, + graph: Any, + text: str, + undefined_concepts: Set[rdflib.URIRef], + ): + self.conforms = conforms + self.graph = graph + self.text = text + self.undefined_concepts = undefined_concepts def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: @@ -229,13 +236,12 @@ def validate( # Relieve RAM of the data graph after validation has run. del data_graph - result = ValidationResult() - result.conforms = validate_result[0] - result.graph = validate_result[1] - result.text = validate_result[2] - result.undefined_concepts = undefined_cdo_concepts - - return result + return ValidationResult( + validate_result[0], + validate_result[1], + validate_result[2], + undefined_cdo_concepts, + ) def main() -> None: From cfe406491b4375512ff97b7181679852578313d3 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 25 Jul 2023 09:02:55 -0400 Subject: [PATCH 12/29] Expose unrecognized-concept set construction logic as function AJN: This is a partial application of @kchason 's work in PR 118, and is being pulled into its own patch series to focus review. References: * https://github.com/casework/CASE-Utilities-Python/pull/118 Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 135 +++++++++++++++------------ 1 file changed, 75 insertions(+), 60 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 6deaaa0..db76636 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -71,6 +71,79 @@ def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: ) or concept_iri.startswith("https://ontology.caseontology.org/") +def get_invalid_cdo_concepts( + data_graph: rdflib.Graph, ontology_graph: rdflib.Graph +) -> Set[rdflib.URIRef]: + """ + Get the set of concepts in the data graph that are not part of the CDO ontology. + + :param data_graph: The data graph to validate. + :param ontology_graph: The ontology graph to use for validation. + :return: The list of concepts in the data graph that are not part of the CDO ontology. + """ + # Construct set of CDO concepts for data graph concept-existence review. + cdo_concepts: Set[rdflib.URIRef] = set() + + for n_structural_class in [ + NS_OWL.Class, + NS_OWL.AnnotationProperty, + NS_OWL.DatatypeProperty, + NS_OWL.ObjectProperty, + NS_RDFS.Datatype, + NS_SH.NodeShape, + NS_SH.PropertyShape, + NS_SH.Shape, + ]: + for ontology_triple in ontology_graph.triples( + (None, NS_RDF.type, n_structural_class) + ): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + if concept_is_cdo_concept(ontology_triple[0]): + cdo_concepts.add(ontology_triple[0]) + for n_ontology_predicate in [ + NS_OWL.backwardCompatibleWith, + NS_OWL.imports, + NS_OWL.incompatibleWith, + NS_OWL.priorVersion, + NS_OWL.versionIRI, + ]: + for ontology_triple in ontology_graph.triples( + (None, n_ontology_predicate, None) + ): + assert isinstance(ontology_triple[0], rdflib.URIRef) + assert isinstance(ontology_triple[2], rdflib.URIRef) + cdo_concepts.add(ontology_triple[0]) + cdo_concepts.add(ontology_triple[2]) + for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + cdo_concepts.add(ontology_triple[0]) + + # Also load historical ontology and version IRIs. + ontology_and_version_iris_data = importlib.resources.read_text( + case_utils.ontology, "ontology_and_version_iris.txt" + ) + for line in ontology_and_version_iris_data.split("\n"): + cleaned_line = line.strip() + if cleaned_line == "": + continue + cdo_concepts.add(rdflib.URIRef(cleaned_line)) + + data_cdo_concepts: Set[rdflib.URIRef] = set() + for data_triple in data_graph.triples((None, None, None)): + for data_triple_member in data_triple: + if isinstance(data_triple_member, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member): + data_cdo_concepts.add(data_triple_member) + elif isinstance(data_triple_member, rdflib.Literal): + if isinstance(data_triple_member.datatype, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member.datatype): + data_cdo_concepts.add(data_triple_member.datatype) + + return data_cdo_concepts - cdo_concepts + + def main() -> None: parser = argparse.ArgumentParser( description="CASE wrapper to pySHACL command line tool." @@ -181,67 +254,9 @@ def main() -> None: _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) ontology_graph.parse(arg_ontology_graph) - # Construct set of CDO concepts for data graph concept-existence review. - cdo_concepts: Set[rdflib.URIRef] = set() - - for n_structural_class in [ - NS_OWL.Class, - NS_OWL.AnnotationProperty, - NS_OWL.DatatypeProperty, - NS_OWL.ObjectProperty, - NS_RDFS.Datatype, - NS_SH.NodeShape, - NS_SH.PropertyShape, - NS_SH.Shape, - ]: - for ontology_triple in ontology_graph.triples( - (None, NS_RDF.type, n_structural_class) - ): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - if concept_is_cdo_concept(ontology_triple[0]): - cdo_concepts.add(ontology_triple[0]) - for n_ontology_predicate in [ - NS_OWL.backwardCompatibleWith, - NS_OWL.imports, - NS_OWL.incompatibleWith, - NS_OWL.priorVersion, - NS_OWL.versionIRI, - ]: - for ontology_triple in ontology_graph.triples( - (None, n_ontology_predicate, None) - ): - assert isinstance(ontology_triple[0], rdflib.URIRef) - assert isinstance(ontology_triple[2], rdflib.URIRef) - cdo_concepts.add(ontology_triple[0]) - cdo_concepts.add(ontology_triple[2]) - for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - cdo_concepts.add(ontology_triple[0]) - - # Also load historical ontology and version IRIs. - ontology_and_version_iris_data = importlib.resources.read_text( - case_utils.ontology, "ontology_and_version_iris.txt" - ) - for line in ontology_and_version_iris_data.split("\n"): - cleaned_line = line.strip() - if cleaned_line == "": - continue - cdo_concepts.add(rdflib.URIRef(cleaned_line)) - - data_cdo_concepts: Set[rdflib.URIRef] = set() - for data_triple in data_graph.triples((None, None, None)): - for data_triple_member in data_triple: - if isinstance(data_triple_member, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member): - data_cdo_concepts.add(data_triple_member) - elif isinstance(data_triple_member, rdflib.Literal): - if isinstance(data_triple_member.datatype, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member.datatype): - data_cdo_concepts.add(data_triple_member.datatype) + # Get the list of undefined CDO concepts in the graph + undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) - undefined_cdo_concepts = data_cdo_concepts - cdo_concepts for undefined_cdo_concept in sorted(undefined_cdo_concepts): warnings.warn(undefined_cdo_concept, NonExistentCDOConceptWarning) undefined_cdo_concepts_message = ( From 1c636e03e8c53d9da26d1282ea173148c8ad959e Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 9 Aug 2023 09:02:52 -0400 Subject: [PATCH 13/29] Add doctest for get_invalid_cdo_concepts Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index db76636..f7db9ab 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -75,11 +75,36 @@ def get_invalid_cdo_concepts( data_graph: rdflib.Graph, ontology_graph: rdflib.Graph ) -> Set[rdflib.URIRef]: """ - Get the set of concepts in the data graph that are not part of the CDO ontology. + Get the set of concepts in the data graph that are not part of the CDO ontologies as specified with the ontology_graph argument. :param data_graph: The data graph to validate. :param ontology_graph: The ontology graph to use for validation. :return: The list of concepts in the data graph that are not part of the CDO ontology. + + >>> from case_utils.namespace import NS_RDF, NS_OWL, NS_UCO_CORE + >>> from rdflib import Graph, Literal, Namespace, URIRef + >>> # Define a namespace for a knowledge base, and a namespace for custom extensions. + >>> ns_kb = Namespace("http://example.org/kb/") + >>> ns_ex = Namespace("http://example.org/ontology/") + >>> dg = Graph() + >>> og = Graph() + >>> # Use an ontology graph in review that includes only a single class and a single property excerpted from UCO, but also a single custom property. + >>> _ = og.add((NS_UCO_CORE.UcoObject, NS_RDF.type, NS_OWL.Class)) + >>> _ = og.add((NS_UCO_CORE.name, NS_RDF.type, NS_OWL.DatatypeProperty)) + >>> _ = og.add((ns_ex.ourCustomProperty, NS_RDF.type, NS_OWL.DatatypeProperty)) + >>> # Define an individual. + >>> n_uco_object = ns_kb["UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c"] + >>> n_uco_object + rdflib.term.URIRef('http://example.org/kb/UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c') + >>> # Review a data graph that includes only the single individual, class typo'd (capitalized incorrectly), but property OK. + >>> _ = dg.add((n_uco_object, NS_RDF.type, NS_UCO_CORE.UCOObject)) + >>> _ = dg.add((n_uco_object, NS_UCO_CORE.name, Literal("Test"))) + >>> _ = dg.add((n_uco_object, ns_ex.customProperty, Literal("Custom Value"))) + >>> invalid_cdo_concepts = get_invalid_cdo_concepts(dg, og) + >>> invalid_cdo_concepts + {rdflib.term.URIRef('https://ontology.unifiedcyberontology.org/uco/core/UCOObject')} + >>> # Note that the property "ourCustomProperty" was typo'd in the data graph, but this was not reported. + >>> assert ns_ex.ourCustomProperty not in invalid_cdo_concepts """ # Construct set of CDO concepts for data graph concept-existence review. cdo_concepts: Set[rdflib.URIRef] = set() From e9dff7b8a8654a6a3d8142f8e2ae7e9cacd9ec77 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 9 Aug 2023 15:57:36 -0400 Subject: [PATCH 14/29] Move doctests checking to own target in preorder traversal As spelled before this patch, the doctests were running after the more time-consuming `case_utils` descent. This lets the tests be called on their own, and also lets a failing doctest fail the CI job quicker. Signed-off-by: Alex Nelson --- tests/Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index f38243d..37ad7e3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -23,6 +23,7 @@ all: \ .PHONY: \ all-case_utils \ check-case_utils \ + check-doctest \ check-isomorphic_diff \ check-mypy \ download @@ -57,13 +58,9 @@ all-case_utils: \ # These check calls are provided in preferred run-order. check: \ check-mypy \ + check-doctest \ check-isomorphic_diff \ check-case_utils - source venv/bin/activate \ - && pytest \ - --doctest-modules \ - --log-level=DEBUG \ - $(top_srcdir)/case_utils source venv/bin/activate \ && pytest \ --ignore case_utils \ @@ -76,6 +73,14 @@ check-case_utils: \ --directory case_utils \ check +check-doctest: \ + .venv.done.log + source venv/bin/activate \ + && pytest \ + --doctest-modules \ + --log-level=DEBUG \ + $(top_srcdir)/case_utils + check-isomorphic_diff: \ .venv.done.log $(MAKE) \ From 40be3114493446a772b563534f7938e6073385cf Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Tue, 15 Aug 2023 11:13:34 -0400 Subject: [PATCH 15/29] Fix None vs "none" ontology version specification No effects were observed on Make-managed files. Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 52369a7..b3c062c 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -98,22 +98,26 @@ def get_ontology_graph( """ Get the ontology graph for the given case_version and any supplemental graphs. - :param case_version: the version of the CASE ontology to use. If None, the most recent version will be used. + :param case_version: the version of the CASE ontology to use. If None (i.e. null), the most recent version will be used. If "none" (the string), no pre-built version of CASE will be used. :param supplemental_graphs: a list of supplemental graphs to use. If None, no supplemental graphs will be used. :return: the ontology graph against which to validate the data graph. """ - if not case_version or case_version == "none": - case_version = CURRENT_CASE_VERSION - ontology_graph = rdflib.Graph() - ttl_filename = case_version + ".ttl" - _logger.debug("ttl_filename = %r.", ttl_filename) - ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) - ontology_graph.parse(data=ttl_data, format="turtle") + + if case_version != "none": + # Load bundled CASE ontology at requested version. + if case_version is None: + case_version = CURRENT_CASE_VERSION + ttl_filename = case_version + ".ttl" + _logger.debug("ttl_filename = %r.", ttl_filename) + ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) + ontology_graph.parse(data=ttl_data, format="turtle") + if supplemental_graphs: for arg_ontology_graph in supplemental_graphs: _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) ontology_graph.parse(arg_ontology_graph) + return ontology_graph From ae5f0771ed84f4600688869ec2f48beedbcf3c30 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Mon, 21 Aug 2023 11:31:07 -0400 Subject: [PATCH 16/29] Add explicit `-> None` on `__init__` It seems to me like `--disallow-untyped-defs`, enabled in `mypy --strict`, should have flagged this as an error. However, from documentation on `no-untyped-def`, `mypy` only requires `__init__(...) -> None` when there is any argument aside from the first `self`. This patch follows the parenthetical recommendation from PEP 484 that `-> None` be given anyways. References: * https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-disallow-untyped-defs * https://mypy.readthedocs.io/en/stable/error_code_list2.html#check-that-every-function-has-an-annotation-no-untyped-def * https://peps.python.org/pep-0484/#the-meaning-of-annotations Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index b3c062c..1c29d6a 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -72,7 +72,7 @@ def __init__( graph: Any, text: str, undefined_concepts: Set[rdflib.URIRef], - ): + ) -> None: self.conforms = conforms self.graph = graph self.text = text From 97d7fbbe4af2f5693af0150d415232ae5e98f898 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Mon, 21 Aug 2023 11:34:52 -0400 Subject: [PATCH 17/29] Constrain `ValidationResult.graph` type to `pyshacl.validate(...)[1]` type Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 1c29d6a..b1283c3 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -37,7 +37,7 @@ import os import sys import warnings -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -69,7 +69,7 @@ class ValidationResult: def __init__( self, conforms: bool, - graph: Any, + graph: Union[Exception, bytes, str, rdflib.Graph], text: str, undefined_concepts: Set[rdflib.URIRef], ) -> None: From 7353c54b608b5c14ba965573c928ac575650dc79 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Mon, 21 Aug 2023 13:51:14 -0400 Subject: [PATCH 18/29] case_validate: Default to None rather than "none" Defaulting to the string `"none"` has a functional consequence in downstream pySHACL operations, as encoded at the time of this writing (pySHACL version 0.23.0). `inference` in the `pyshacl.validate(...)` arguments is propagated down two code paths that both impact the validation operation, but that use different default values. The `--metashacl` (SHACL-SHACL validation graph for reviewing shape syntax)- aligned code path defaults to `rdfs` inference, and the `Validator` class defaults to the string `"none"`. Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 6deaaa0..9fa30ce 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -136,12 +136,12 @@ def main() -> None: action="store_true", help="(As with pyshacl CLI) Allow import of sub-graphs defined in statements with owl:imports.", ) + # NOTE: The "ontology graph" in the --inference help is the mix of the SHACL shapes graph and OWL ontology (or RDFS schema) graph. parser.add_argument( "-i", "--inference", choices=("none", "rdfs", "owlrl", "both"), - default="none", - help='(As with pyshacl CLI) Choose a type of inferencing to run against the Data Graph before validating. Default is "none".', + help='(As with pyshacl CLI) Choose a type of inferencing to run against the Data Graph before validating. The default behavior if this flag is not provided is to behave as "none", if not using the --metashacl flag. The default behavior when using the --metashacl flag will apply "rdfs" inferencing to the ontology graph, but the data graph will still have no inferencing applied. If the --inference flag is provided, it will apply to both the ontology graph, and the data graph.', ) parser.add_argument( "-m", From 4208a99eab30330d20d1037bab0f52f086944da6 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 22 Aug 2023 13:50:50 +0200 Subject: [PATCH 19/29] Wrap errors and positional arg signature support --- case_utils/case_validate/__init__.py | 35 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 1886583..108b515 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -37,7 +37,7 @@ import os import sys import warnings -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Union import pyshacl # type: ignore import rdflib @@ -65,6 +65,14 @@ class NonExistentCDOConceptWarning(UserWarning): pass +class NonExistentCASEVersionError(Exception): + """ + This class is used when an invalid CASE version is requested that is not supported by the library. + """ + + pass + + class ValidationResult: def __init__( self, @@ -204,10 +212,21 @@ def get_ontology_graph( if case_version != "none": # Load bundled CASE ontology at requested version. - if case_version is None: + if case_version is None or case_version == "": case_version = CURRENT_CASE_VERSION + # If the first character case_version is numeric, prepend case- to it. This allows for the version to be passed + # by the library as both case-1.2.0 and 1.2.0 + if case_version[0].isdigit(): + case_version = "case-" + case_version ttl_filename = case_version + ".ttl" _logger.debug("ttl_filename = %r.", ttl_filename) + # Ensure the requested version of the CASE ontology is available and if not, throw an appropriate exception + # that can be returned in a user-friendly message. + if not importlib.resources.is_resource(case_utils.ontology, ttl_filename): + raise NonExistentCASEVersionError( + f"The requested version ({case_version}) of the CASE ontology is not available. Please choose a " + f"different version. The latest supported version is: {CURRENT_CASE_VERSION}" + ) ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) ontology_graph.parse(data=ttl_data, format="turtle") @@ -221,19 +240,23 @@ def get_ontology_graph( def validate( input_file: str, + *args: Any, case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None, abort_on_first: bool = False, inference: Optional[str] = "none", + **kwargs: Any, ) -> ValidationResult: """ Validate the given data graph against the given CASE ontology version and supplemental graphs. - + :param *args: The positional arguments to pass to the underlying pyshacl.validate function. :param input_file: The path to the file containing the data graph to validate. - :param case_version: The version of the CASE ontology to use. If None, the most recent version will be used. + :param case_version: The version of the CASE ontology to use (e.g. 1.2.0). If None, the most recent version will + be used. :param supplemental_graphs: The supplemental graphs to use. If None, no supplemental graphs will be used. :param abort_on_first: Whether to abort on the first validation error. :param inference: The type of inference to use. If "none", no inference will be used. + :param **kwargs: The keyword arguments to pass to the underlying pyshacl.validate function. :return: The validation result object containing the defined properties. """ # Convert the data graph string to a rdflib.Graph object. @@ -260,6 +283,8 @@ def validate( allow_warnings=False, debug=False, do_owl_imports=False, + args=args, + kwargs=kwargs, ) # Relieve RAM of the data graph after validation has run. @@ -409,7 +434,7 @@ def main() -> None: allow_warnings=True if args.allow_warnings else False, debug=True if args.debug else False, do_owl_imports=True if args.imports else False, - **validator_kwargs + **validator_kwargs, ) # Relieve RAM of the data graph after validation has run. From 73e468366f83b7dd72400d50e2d3d774d4f0b146 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 22 Aug 2023 14:00:59 +0200 Subject: [PATCH 20/29] Separate types and utils into discrete files --- case_utils/case_validate/__init__.py | 198 +-------------------- case_utils/case_validate/validate_types.py | 33 ++++ case_utils/case_validate/validate_utils.py | 168 +++++++++++++++++ 3 files changed, 210 insertions(+), 189 deletions(-) create mode 100644 case_utils/case_validate/validate_types.py create mode 100644 case_utils/case_validate/validate_utils.py diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 108b515..48ad26e 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -32,212 +32,32 @@ __version__ = "0.3.0" import argparse -import importlib.resources import logging import os import sys import warnings -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import pyshacl # type: ignore import rdflib from rdflib import Graph -import case_utils.ontology +from case_utils.case_validate.validate_types import ( + NonExistentCDOConceptWarning, + ValidationResult, +) +from case_utils.case_validate.validate_utils import ( + get_invalid_cdo_concepts, + get_ontology_graph, +) from case_utils.ontology.version_info import ( CURRENT_CASE_VERSION, built_version_choices_list, ) -NS_OWL = rdflib.OWL -NS_RDF = rdflib.RDF -NS_RDFS = rdflib.RDFS -NS_SH = rdflib.SH - _logger = logging.getLogger(os.path.basename(__file__)) -class NonExistentCDOConceptWarning(UserWarning): - """ - This class is used when a concept is encountered in the data graph that is not part of CDO ontologies, according to the --built-version flags and --ontology-graph flags. - """ - - pass - - -class NonExistentCASEVersionError(Exception): - """ - This class is used when an invalid CASE version is requested that is not supported by the library. - """ - - pass - - -class ValidationResult: - def __init__( - self, - conforms: bool, - graph: Union[Exception, bytes, str, rdflib.Graph], - text: str, - undefined_concepts: Set[rdflib.URIRef], - ) -> None: - self.conforms = conforms - self.graph = graph - self.text = text - self.undefined_concepts = undefined_concepts - - -def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: - """ - Determine if a concept is part of the CDO ontology. - - :param n_concept: The concept to check. - :return: whether the concept is part of the CDO ontologies. - """ - concept_iri = str(n_concept) - return concept_iri.startswith( - "https://ontology.unifiedcyberontology.org/" - ) or concept_iri.startswith("https://ontology.caseontology.org/") - - -def get_invalid_cdo_concepts( - data_graph: rdflib.Graph, ontology_graph: rdflib.Graph -) -> Set[rdflib.URIRef]: - """ - Get the set of concepts in the data graph that are not part of the CDO ontologies as specified with the ontology_graph argument. - - :param data_graph: The data graph to validate. - :param ontology_graph: The ontology graph to use for validation. - :return: The list of concepts in the data graph that are not part of the CDO ontology. - - >>> from case_utils.namespace import NS_RDF, NS_OWL, NS_UCO_CORE - >>> from rdflib import Graph, Literal, Namespace, URIRef - >>> # Define a namespace for a knowledge base, and a namespace for custom extensions. - >>> ns_kb = Namespace("http://example.org/kb/") - >>> ns_ex = Namespace("http://example.org/ontology/") - >>> dg = Graph() - >>> og = Graph() - >>> # Use an ontology graph in review that includes only a single class and a single property excerpted from UCO, but also a single custom property. - >>> _ = og.add((NS_UCO_CORE.UcoObject, NS_RDF.type, NS_OWL.Class)) - >>> _ = og.add((NS_UCO_CORE.name, NS_RDF.type, NS_OWL.DatatypeProperty)) - >>> _ = og.add((ns_ex.ourCustomProperty, NS_RDF.type, NS_OWL.DatatypeProperty)) - >>> # Define an individual. - >>> n_uco_object = ns_kb["UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c"] - >>> n_uco_object - rdflib.term.URIRef('http://example.org/kb/UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c') - >>> # Review a data graph that includes only the single individual, class typo'd (capitalized incorrectly), but property OK. - >>> _ = dg.add((n_uco_object, NS_RDF.type, NS_UCO_CORE.UCOObject)) - >>> _ = dg.add((n_uco_object, NS_UCO_CORE.name, Literal("Test"))) - >>> _ = dg.add((n_uco_object, ns_ex.customProperty, Literal("Custom Value"))) - >>> invalid_cdo_concepts = get_invalid_cdo_concepts(dg, og) - >>> invalid_cdo_concepts - {rdflib.term.URIRef('https://ontology.unifiedcyberontology.org/uco/core/UCOObject')} - >>> # Note that the property "ourCustomProperty" was typo'd in the data graph, but this was not reported. - >>> assert ns_ex.ourCustomProperty not in invalid_cdo_concepts - """ - # Construct set of CDO concepts for data graph concept-existence review. - cdo_concepts: Set[rdflib.URIRef] = set() - - for n_structural_class in [ - NS_OWL.Class, - NS_OWL.AnnotationProperty, - NS_OWL.DatatypeProperty, - NS_OWL.ObjectProperty, - NS_RDFS.Datatype, - NS_SH.NodeShape, - NS_SH.PropertyShape, - NS_SH.Shape, - ]: - for ontology_triple in ontology_graph.triples( - (None, NS_RDF.type, n_structural_class) - ): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - if concept_is_cdo_concept(ontology_triple[0]): - cdo_concepts.add(ontology_triple[0]) - for n_ontology_predicate in [ - NS_OWL.backwardCompatibleWith, - NS_OWL.imports, - NS_OWL.incompatibleWith, - NS_OWL.priorVersion, - NS_OWL.versionIRI, - ]: - for ontology_triple in ontology_graph.triples( - (None, n_ontology_predicate, None) - ): - assert isinstance(ontology_triple[0], rdflib.URIRef) - assert isinstance(ontology_triple[2], rdflib.URIRef) - cdo_concepts.add(ontology_triple[0]) - cdo_concepts.add(ontology_triple[2]) - for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): - if not isinstance(ontology_triple[0], rdflib.URIRef): - continue - cdo_concepts.add(ontology_triple[0]) - - # Also load historical ontology and version IRIs. - ontology_and_version_iris_data = importlib.resources.read_text( - case_utils.ontology, "ontology_and_version_iris.txt" - ) - for line in ontology_and_version_iris_data.split("\n"): - cleaned_line = line.strip() - if cleaned_line == "": - continue - cdo_concepts.add(rdflib.URIRef(cleaned_line)) - - data_cdo_concepts: Set[rdflib.URIRef] = set() - for data_triple in data_graph.triples((None, None, None)): - for data_triple_member in data_triple: - if isinstance(data_triple_member, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member): - data_cdo_concepts.add(data_triple_member) - elif isinstance(data_triple_member, rdflib.Literal): - if isinstance(data_triple_member.datatype, rdflib.URIRef): - if concept_is_cdo_concept(data_triple_member.datatype): - data_cdo_concepts.add(data_triple_member.datatype) - - return data_cdo_concepts - cdo_concepts - - -def get_ontology_graph( - case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None -) -> rdflib.Graph: - """ - Get the ontology graph for the given case_version and any supplemental graphs. - - :param case_version: the version of the CASE ontology to use. If None (i.e. null), the most recent version will be used. If "none" (the string), no pre-built version of CASE will be used. - :param supplemental_graphs: a list of supplemental graphs to use. If None, no supplemental graphs will be used. - :return: the ontology graph against which to validate the data graph. - """ - ontology_graph = rdflib.Graph() - - if case_version != "none": - # Load bundled CASE ontology at requested version. - if case_version is None or case_version == "": - case_version = CURRENT_CASE_VERSION - # If the first character case_version is numeric, prepend case- to it. This allows for the version to be passed - # by the library as both case-1.2.0 and 1.2.0 - if case_version[0].isdigit(): - case_version = "case-" + case_version - ttl_filename = case_version + ".ttl" - _logger.debug("ttl_filename = %r.", ttl_filename) - # Ensure the requested version of the CASE ontology is available and if not, throw an appropriate exception - # that can be returned in a user-friendly message. - if not importlib.resources.is_resource(case_utils.ontology, ttl_filename): - raise NonExistentCASEVersionError( - f"The requested version ({case_version}) of the CASE ontology is not available. Please choose a " - f"different version. The latest supported version is: {CURRENT_CASE_VERSION}" - ) - ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) - ontology_graph.parse(data=ttl_data, format="turtle") - - if supplemental_graphs: - for arg_ontology_graph in supplemental_graphs: - _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) - ontology_graph.parse(arg_ontology_graph) - - return ontology_graph - - def validate( input_file: str, *args: Any, diff --git a/case_utils/case_validate/validate_types.py b/case_utils/case_validate/validate_types.py new file mode 100644 index 0000000..b1911aa --- /dev/null +++ b/case_utils/case_validate/validate_types.py @@ -0,0 +1,33 @@ +from typing import Set, Union + +import rdflib + + +class ValidationResult: + def __init__( + self, + conforms: bool, + graph: Union[Exception, bytes, str, rdflib.Graph], + text: str, + undefined_concepts: Set[rdflib.URIRef], + ) -> None: + self.conforms = conforms + self.graph = graph + self.text = text + self.undefined_concepts = undefined_concepts + + +class NonExistentCDOConceptWarning(UserWarning): + """ + This class is used when a concept is encountered in the data graph that is not part of CDO ontologies, according to the --built-version flags and --ontology-graph flags. + """ + + pass + + +class NonExistentCASEVersionError(Exception): + """ + This class is used when an invalid CASE version is requested that is not supported by the library. + """ + + pass diff --git a/case_utils/case_validate/validate_utils.py b/case_utils/case_validate/validate_utils.py new file mode 100644 index 0000000..a53e92c --- /dev/null +++ b/case_utils/case_validate/validate_utils.py @@ -0,0 +1,168 @@ +import importlib +import logging +import os +from typing import List, Optional, Set + +import rdflib + +import case_utils +from case_utils.case_validate.validate_types import NonExistentCASEVersionError +from case_utils.ontology import CURRENT_CASE_VERSION + +NS_OWL = rdflib.OWL +NS_RDF = rdflib.RDF +NS_RDFS = rdflib.RDFS +NS_SH = rdflib.SH + +_logger = logging.getLogger(os.path.basename(__file__)) + + +def concept_is_cdo_concept(n_concept: rdflib.URIRef) -> bool: + """ + Determine if a concept is part of the CDO ontology. + + :param n_concept: The concept to check. + :return: whether the concept is part of the CDO ontologies. + """ + concept_iri = str(n_concept) + return concept_iri.startswith( + "https://ontology.unifiedcyberontology.org/" + ) or concept_iri.startswith("https://ontology.caseontology.org/") + + +def get_invalid_cdo_concepts( + data_graph: rdflib.Graph, ontology_graph: rdflib.Graph +) -> Set[rdflib.URIRef]: + """ + Get the set of concepts in the data graph that are not part of the CDO ontologies as specified with the ontology_graph argument. + + :param data_graph: The data graph to validate. + :param ontology_graph: The ontology graph to use for validation. + :return: The list of concepts in the data graph that are not part of the CDO ontology. + + >>> from case_utils.namespace import NS_RDF, NS_OWL, NS_UCO_CORE + >>> from rdflib import Graph, Literal, Namespace, URIRef + >>> # Define a namespace for a knowledge base, and a namespace for custom extensions. + >>> ns_kb = Namespace("http://example.org/kb/") + >>> ns_ex = Namespace("http://example.org/ontology/") + >>> dg = Graph() + >>> og = Graph() + >>> # Use an ontology graph in review that includes only a single class and a single property excerpted from UCO, but also a single custom property. + >>> _ = og.add((NS_UCO_CORE.UcoObject, NS_RDF.type, NS_OWL.Class)) + >>> _ = og.add((NS_UCO_CORE.name, NS_RDF.type, NS_OWL.DatatypeProperty)) + >>> _ = og.add((ns_ex.ourCustomProperty, NS_RDF.type, NS_OWL.DatatypeProperty)) + >>> # Define an individual. + >>> n_uco_object = ns_kb["UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c"] + >>> n_uco_object + rdflib.term.URIRef('http://example.org/kb/UcoObject-f494d239-d9fd-48da-bc07-461ba86d8c6c') + >>> # Review a data graph that includes only the single individual, class typo'd (capitalized incorrectly), but property OK. + >>> _ = dg.add((n_uco_object, NS_RDF.type, NS_UCO_CORE.UCOObject)) + >>> _ = dg.add((n_uco_object, NS_UCO_CORE.name, Literal("Test"))) + >>> _ = dg.add((n_uco_object, ns_ex.customProperty, Literal("Custom Value"))) + >>> invalid_cdo_concepts = get_invalid_cdo_concepts(dg, og) + >>> invalid_cdo_concepts + {rdflib.term.URIRef('https://ontology.unifiedcyberontology.org/uco/core/UCOObject')} + >>> # Note that the property "ourCustomProperty" was typo'd in the data graph, but this was not reported. + >>> assert ns_ex.ourCustomProperty not in invalid_cdo_concepts + """ + # Construct set of CDO concepts for data graph concept-existence review. + cdo_concepts: Set[rdflib.URIRef] = set() + + for n_structural_class in [ + NS_OWL.Class, + NS_OWL.AnnotationProperty, + NS_OWL.DatatypeProperty, + NS_OWL.ObjectProperty, + NS_RDFS.Datatype, + NS_SH.NodeShape, + NS_SH.PropertyShape, + NS_SH.Shape, + ]: + for ontology_triple in ontology_graph.triples( + (None, NS_RDF.type, n_structural_class) + ): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + if concept_is_cdo_concept(ontology_triple[0]): + cdo_concepts.add(ontology_triple[0]) + for n_ontology_predicate in [ + NS_OWL.backwardCompatibleWith, + NS_OWL.imports, + NS_OWL.incompatibleWith, + NS_OWL.priorVersion, + NS_OWL.versionIRI, + ]: + for ontology_triple in ontology_graph.triples( + (None, n_ontology_predicate, None) + ): + assert isinstance(ontology_triple[0], rdflib.URIRef) + assert isinstance(ontology_triple[2], rdflib.URIRef) + cdo_concepts.add(ontology_triple[0]) + cdo_concepts.add(ontology_triple[2]) + for ontology_triple in ontology_graph.triples((None, NS_RDF.type, NS_OWL.Ontology)): + if not isinstance(ontology_triple[0], rdflib.URIRef): + continue + cdo_concepts.add(ontology_triple[0]) + + # Also load historical ontology and version IRIs. + ontology_and_version_iris_data = importlib.resources.read_text( + case_utils.ontology, "ontology_and_version_iris.txt" + ) + for line in ontology_and_version_iris_data.split("\n"): + cleaned_line = line.strip() + if cleaned_line == "": + continue + cdo_concepts.add(rdflib.URIRef(cleaned_line)) + + data_cdo_concepts: Set[rdflib.URIRef] = set() + for data_triple in data_graph.triples((None, None, None)): + for data_triple_member in data_triple: + if isinstance(data_triple_member, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member): + data_cdo_concepts.add(data_triple_member) + elif isinstance(data_triple_member, rdflib.Literal): + if isinstance(data_triple_member.datatype, rdflib.URIRef): + if concept_is_cdo_concept(data_triple_member.datatype): + data_cdo_concepts.add(data_triple_member.datatype) + + return data_cdo_concepts - cdo_concepts + + +def get_ontology_graph( + case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None +) -> rdflib.Graph: + """ + Get the ontology graph for the given case_version and any supplemental graphs. + + :param case_version: the version of the CASE ontology to use. If None (i.e. null), the most recent version will be used. If "none" (the string), no pre-built version of CASE will be used. + :param supplemental_graphs: a list of supplemental graphs to use. If None, no supplemental graphs will be used. + :return: the ontology graph against which to validate the data graph. + """ + ontology_graph = rdflib.Graph() + + if case_version != "none": + # Load bundled CASE ontology at requested version. + if case_version is None or case_version == "": + case_version = CURRENT_CASE_VERSION + # If the first character case_version is numeric, prepend case- to it. This allows for the version to be passed + # by the library as both case-1.2.0 and 1.2.0 + if case_version[0].isdigit(): + case_version = "case-" + case_version + ttl_filename = case_version + ".ttl" + _logger.debug("ttl_filename = %r.", ttl_filename) + # Ensure the requested version of the CASE ontology is available and if not, throw an appropriate exception + # that can be returned in a user-friendly message. + if not importlib.resources.is_resource(case_utils.ontology, ttl_filename): + raise NonExistentCASEVersionError( + f"The requested version ({case_version}) of the CASE ontology is not available. Please choose a " + f"different version. The latest supported version is: {CURRENT_CASE_VERSION}" + ) + ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) + ontology_graph.parse(data=ttl_data, format="turtle") + + if supplemental_graphs: + for arg_ontology_graph in supplemental_graphs: + _logger.debug("arg_ontology_graph = %r.", arg_ontology_graph) + ontology_graph.parse(arg_ontology_graph) + + return ontology_graph From d41a418a50560cc9fa6142bf1c14e3b3987afacf Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 22 Aug 2023 14:12:14 +0200 Subject: [PATCH 21/29] Fix import reference --- case_utils/case_validate/validate_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/case_utils/case_validate/validate_utils.py b/case_utils/case_validate/validate_utils.py index a53e92c..e658e05 100644 --- a/case_utils/case_validate/validate_utils.py +++ b/case_utils/case_validate/validate_utils.py @@ -7,7 +7,7 @@ import case_utils from case_utils.case_validate.validate_types import NonExistentCASEVersionError -from case_utils.ontology import CURRENT_CASE_VERSION +from case_utils.ontology.version_info import CURRENT_CASE_VERSION NS_OWL = rdflib.OWL NS_RDF = rdflib.RDF From 8f957dc029c78b1ca31404a2b97d2c0872a9d647 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 23 Aug 2023 10:20:36 -0400 Subject: [PATCH 22/29] Forward arguments with unpacking syntax Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 48ad26e..838ff17 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -94,6 +94,7 @@ def validate( bool, Union[Exception, bytes, str, rdflib.Graph], str ] = pyshacl.validate( data_graph, + *args, shacl_graph=ontology_graph, ont_graph=ontology_graph, inference=inference, @@ -103,8 +104,7 @@ def validate( allow_warnings=False, debug=False, do_owl_imports=False, - args=args, - kwargs=kwargs, + **kwargs, ) # Relieve RAM of the data graph after validation has run. From 00f1360906f8bb533d3ef0c886a43fd0371150b3 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 23 Aug 2023 10:39:53 -0400 Subject: [PATCH 23/29] Default case_validate.validate inference parameter to None rather than "none" This is a continuation of PR 123. References: * https://github.com/casework/CASE-Utilities-Python/pull/123 Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index addf460..8bd32c7 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -64,7 +64,7 @@ def validate( case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None, abort_on_first: bool = False, - inference: Optional[str] = "none", + inference: Optional[str] = None, **kwargs: Any, ) -> ValidationResult: """ @@ -75,7 +75,7 @@ def validate( be used. :param supplemental_graphs: The supplemental graphs to use. If None, no supplemental graphs will be used. :param abort_on_first: Whether to abort on the first validation error. - :param inference: The type of inference to use. If "none", no inference will be used. + :param inference: The type of inference to use. If "none" (type str), no inference will be used. If None (type NoneType), pyshacl defaults will be used. Note that at the time of this writing (pySHACL 0.23.0), pyshacl defaults are no inferencing for the data graph, and RDFS inferencing for the SHACL graph, which for case_utils.validate includes the SHACL and OWL graphs. :param **kwargs: The keyword arguments to pass to the underlying pyshacl.validate function. :return: The validation result object containing the defined properties. """ From 15f00c9cb6711b72fdc35a2ef39788f2981efef9 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 23 Aug 2023 17:41:47 -0400 Subject: [PATCH 24/29] Consolidate case_validate CLI validation logic into case_validate.validate This patch separates implementation points between functionality distinct to `case_utils.validate` and `pyshacl.validate`. The `allow_warnings` and `inference` parameters provide CASE-specific documentation as an augmentation to `pyshacl.validate`'s documentation, but otherwise other documentation on `pyshacl.validate`'s keyword arguments is delegated to their upstream function. This patch removes some hardcoded parameter values in `pyshacl.validate`, letting the `case_validate` CLI or caller provide any runtime-requested values. Also, without functional impact, this patch sorts keyword parameters alphabetically. Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 102 +++++++++++---------------- 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 8bd32c7..d1fc70b 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -59,59 +59,74 @@ def validate( - input_file: str, + input_file: Union[List[str], str], *args: Any, case_version: Optional[str] = None, supplemental_graphs: Optional[List[str]] = None, - abort_on_first: bool = False, - inference: Optional[str] = None, **kwargs: Any, ) -> ValidationResult: """ Validate the given data graph against the given CASE ontology version and supplemental graphs. + :param *args: The positional arguments to pass to the underlying pyshacl.validate function. - :param input_file: The path to the file containing the data graph to validate. - :param case_version: The version of the CASE ontology to use (e.g. 1.2.0). If None, the most recent version will - be used. - :param supplemental_graphs: The supplemental graphs to use. If None, no supplemental graphs will be used. - :param abort_on_first: Whether to abort on the first validation error. + :param input_file: The path to the file containing the data graph to validate. This can also be a list of paths to files containing data graphs to pool together. + :param case_version: The version of the CASE ontology to use (e.g. 1.2.0). If None, the most recent version will be used. + :param supplemental_graphs: File paths to supplemental graphs to use. If None, no supplemental graphs will be used. + :param allow_warnings: In addition to affecting the conformance of SHACL validation, this will affect conformance based on unrecognized CDO concepts (likely, misspelled or miscapitalized) in the data graph. If allow_warnings is not True, any unrecognized concept using a CDO IRI prefix will cause conformance to be False. :param inference: The type of inference to use. If "none" (type str), no inference will be used. If None (type NoneType), pyshacl defaults will be used. Note that at the time of this writing (pySHACL 0.23.0), pyshacl defaults are no inferencing for the data graph, and RDFS inferencing for the SHACL graph, which for case_utils.validate includes the SHACL and OWL graphs. :param **kwargs: The keyword arguments to pass to the underlying pyshacl.validate function. :return: The validation result object containing the defined properties. """ # Convert the data graph string to a rdflib.Graph object. data_graph = rdflib.Graph() - data_graph.parse(input_file) + if isinstance(input_file, str): + data_graph.parse(input_file) + elif isinstance(input_file, list): + for _data_graph_file in input_file: + _logger.debug("_data_graph_file = %r.", _data_graph_file) + if not isinstance(_data_graph_file, str): + raise TypeError("Expected str, received %s." % type(_data_graph_file)) + data_graph.parse(_data_graph_file) # Get the ontology graph from the case_version and supplemental_graphs arguments ontology_graph: Graph = get_ontology_graph(case_version, supplemental_graphs) - # Get the undefined CDO concepts + # Get the undefined CDO concepts. undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) + # Warn about typo'd concepts before performing SHACL review. + for undefined_cdo_concept in sorted(undefined_cdo_concepts): + warnings.warn(undefined_cdo_concept, NonExistentCDOConceptWarning) + undefined_cdo_concepts_message = ( + "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." + % len(undefined_cdo_concepts) + ) + # Validate data graph against ontology graph. validate_result: Tuple[ bool, Union[Exception, bytes, str, rdflib.Graph], str ] = pyshacl.validate( data_graph, *args, - shacl_graph=ontology_graph, ont_graph=ontology_graph, - inference=inference, - meta_shacl=False, - abort_on_first=abort_on_first, - allow_infos=False, - allow_warnings=False, - debug=False, - do_owl_imports=False, + shacl_graph=ontology_graph, **kwargs, ) # Relieve RAM of the data graph after validation has run. del data_graph + conforms = validate_result[0] + + if len(undefined_cdo_concepts) > 0: + warnings.warn(undefined_cdo_concepts_message) + if not kwargs.get("allow_warnings"): + undefined_cdo_concepts_alleviation_message = "The data graph is SHACL-conformant with the CDO ontologies, but nonexistent-concept references raise Warnings with this tool. Please either correct the concept names in the data graph; use the --ontology-graph flag to pass a corrected CDO ontology file, also using --built-version none; or, use the --allow-warnings flag." + warnings.warn(undefined_cdo_concepts_alleviation_message) + conforms = False + return ValidationResult( - validate_result[0], + conforms, validate_result[1], validate_result[2], undefined_cdo_concepts, @@ -212,25 +227,6 @@ def main() -> None: args = parser.parse_args() - data_graph = rdflib.Graph() - for in_graph in args.in_graph: - _logger.debug("in_graph = %r.", in_graph) - data_graph.parse(in_graph) - - # Get the ontology graph based on the CASE version and supplemental graphs specified by the CLI - ontology_graph = get_ontology_graph( - case_version=args.built_version, supplemental_graphs=args.ontology_graph - ) - - # Get the list of undefined CDO concepts in the graph - undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) - for undefined_cdo_concept in sorted(undefined_cdo_concepts): - warnings.warn(undefined_cdo_concept, NonExistentCDOConceptWarning) - undefined_cdo_concepts_message = ( - "There were %d concepts with CDO IRIs in the data graph that are not in the ontology graph." - % len(undefined_cdo_concepts) - ) - # Determine output format. # pySHACL's determination of output formatting is handled solely # through the -f flag. Other CASE CLI tools handle format @@ -241,28 +237,23 @@ def main() -> None: if args.format != "human": validator_kwargs["serialize_report_graph"] = args.format - validate_result: Tuple[ - bool, Union[Exception, bytes, str, rdflib.Graph], str - ] = pyshacl.validate( - data_graph, - shacl_graph=ontology_graph, - ont_graph=ontology_graph, - inference=args.inference, - meta_shacl=args.metashacl, + validation_result: ValidationResult = validate( + args.in_graph, abort_on_first=args.abort, allow_infos=True if args.allow_infos else False, allow_warnings=True if args.allow_warnings else False, + case_version=args.built_version, debug=True if args.debug else False, do_owl_imports=True if args.imports else False, + inference=args.inference, + meta_shacl=args.metashacl, + supplemental_graphs=args.ontology_graph, **validator_kwargs, ) - # Relieve RAM of the data graph after validation has run. - del data_graph - - conforms = validate_result[0] - validation_graph = validate_result[1] - validation_text = validate_result[2] + conforms = validation_result.conforms + validation_graph = validation_result.graph + validation_text = validation_result.text # NOTE: The output logistics code is adapted from pySHACL's file # pyshacl/cli.py. This section should be monitored for code drift. @@ -284,13 +275,6 @@ def main() -> None: % type(validation_graph) ) - if len(undefined_cdo_concepts) > 0: - warnings.warn(undefined_cdo_concepts_message) - if not args.allow_warnings: - undefined_cdo_concepts_alleviation_message = "The data graph is SHACL-conformant with the CDO ontologies, but nonexistent-concept references raise Warnings with this tool. Please either correct the concept names in the data graph; use the --ontology-graph flag to pass a corrected CDO ontology file, also using --built-version none; or, use the --allow-warnings flag." - warnings.warn(undefined_cdo_concepts_alleviation_message) - conforms = False - sys.exit(0 if conforms else 1) From eccaad42a89ef309de3cc993cef988e7a38c14d4 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 23 Aug 2023 17:54:04 -0400 Subject: [PATCH 25/29] Add new case_validate source files to Make dependencies Signed-off-by: Alex Nelson --- tests/case_utils/case_file/Makefile | 3 ++ .../case_validate/case_test_examples/Makefile | 2 ++ tests/case_utils/case_validate/cli/Makefile | 35 +++++++++++-------- .../case_validate/shape_disabling/Makefile | 6 ++++ .../case_validate/uco_test_examples/Makefile | 2 ++ 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/tests/case_utils/case_file/Makefile b/tests/case_utils/case_file/Makefile index 91bf6d6..e2014ec 100644 --- a/tests/case_utils/case_file/Makefile +++ b/tests/case_utils/case_file/Makefile @@ -82,6 +82,9 @@ kb.ttl: \ #TODO - kb.json has a conversion error with context dictionary construction and custom datatypes. kb_validation.ttl: \ + $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ kb.ttl source $(tests_srcdir)/venv/bin/activate \ && case_validate \ diff --git a/tests/case_utils/case_validate/case_test_examples/Makefile b/tests/case_utils/case_validate/case_test_examples/Makefile index 089c840..ff33917 100644 --- a/tests/case_utils/case_validate/case_test_examples/Makefile +++ b/tests/case_utils/case_validate/case_test_examples/Makefile @@ -44,6 +44,8 @@ all: \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f __$@ _$@ source $(tests_srcdir)/venv/bin/activate \ diff --git a/tests/case_utils/case_validate/cli/Makefile b/tests/case_utils/case_validate/cli/Makefile index dc78c50..5795ad4 100644 --- a/tests/case_utils/case_validate/cli/Makefile +++ b/tests/case_utils/case_validate/cli/Makefile @@ -23,6 +23,11 @@ tests_srcdir := $(top_srcdir)/tests RDF_TOOLKIT_JAR := $(case_srcdir)/lib/rdf-toolkit.jar +case_validate_sources := \ + $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py + files_to_generate := \ errant_cdo_concept_PASS.txt \ errant_cdo_concept_XFAIL.txt \ @@ -62,9 +67,9 @@ clean: $(files_to_generate) errant_cdo_concept_PASS.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ errant_cdo_concept.ttl rm -f _$@ @@ -77,9 +82,9 @@ errant_cdo_concept_PASS.txt: \ mv _$@ $@ errant_cdo_concept_XFAIL.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ errant_cdo_concept.ttl rm -f _$@ @@ -94,9 +99,9 @@ errant_cdo_concept_XFAIL.txt: \ format_human_output_%: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -109,9 +114,9 @@ format_human_output_%: \ format_human_output_unspecified.txt: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -126,9 +131,9 @@ format_human_output_unspecified.txt: \ # blank node ID that does not serve to inform the example. format_jsonld_output_%: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f __$@ _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -146,9 +151,9 @@ format_jsonld_output_%: \ format_jsonld_output_unspecified.jsonld: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f __$@ _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -166,9 +171,9 @@ format_jsonld_output_unspecified.jsonld: \ format_turtle_output_%: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -181,9 +186,9 @@ format_turtle_output_%: \ format_turtle_output_unspecified.ttl: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -196,9 +201,9 @@ format_turtle_output_unspecified.ttl: \ format_unspecified_output_%: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -210,9 +215,9 @@ format_unspecified_output_%: \ format_unspecified_output_unspecified.txt: \ $(examples_srcdir)/investigative_action_PASS_validation.ttl \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py rm -f _$@ source $(tests_srcdir)/venv/bin/activate \ @@ -223,9 +228,9 @@ format_unspecified_output_unspecified.txt: \ mv _$@ $@ past_version_reference_PASS.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ $(top_srcdir)/case_utils/ontology/ontology_and_version_iris.txt \ past_version_reference_PASS.ttl @@ -237,9 +242,9 @@ past_version_reference_PASS.txt: \ mv _$@ $@ past_version_reference_XFAIL.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ $(top_srcdir)/case_utils/ontology/ontology_and_version_iris.txt \ past_version_reference_XFAIL.ttl @@ -253,9 +258,9 @@ past_version_reference_XFAIL.txt: \ mv _$@ $@ split_data_graph_PASS.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ split_data_graph_1.json \ split_data_graph_2.json @@ -269,9 +274,9 @@ split_data_graph_PASS.txt: \ mv _$@ $@ split_data_graph_XFAIL.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ split_data_graph_1.json rm -f _$@ @@ -285,9 +290,9 @@ split_data_graph_XFAIL.txt: \ mv _$@ $@ thing_metashacl_PASS.txt: \ + $(case_validate_sources) \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ - $(top_srcdir)/case_utils/case_validate/__init__.py \ $(top_srcdir)/case_utils/ontology/__init__.py \ thing.ttl rm -f _$@ diff --git a/tests/case_utils/case_validate/shape_disabling/Makefile b/tests/case_utils/case_validate/shape_disabling/Makefile index b9603ac..db3437b 100644 --- a/tests/case_utils/case_validate/shape_disabling/Makefile +++ b/tests/case_utils/case_validate/shape_disabling/Makefile @@ -32,6 +32,9 @@ clean: validation_with_uuid_shape_disabled.txt: \ $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ disable_shape.ttl \ example.ttl source $(tests_srcdir)/venv/bin/activate \ @@ -43,6 +46,9 @@ validation_with_uuid_shape_disabled.txt: \ validation_with_uuid_shape_enabled.txt: \ $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ example.ttl source $(tests_srcdir)/venv/bin/activate \ && case_validate \ diff --git a/tests/case_utils/case_validate/uco_test_examples/Makefile b/tests/case_utils/case_validate/uco_test_examples/Makefile index 2d4550b..921b683 100644 --- a/tests/case_utils/case_validate/uco_test_examples/Makefile +++ b/tests/case_utils/case_validate/uco_test_examples/Makefile @@ -76,6 +76,8 @@ all: \ $(tests_srcdir)/.venv.done.log \ $(top_srcdir)/.ontology.done.log \ $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ $(top_srcdir)/case_utils/ontology/__init__.py source $(tests_srcdir)/venv/bin/activate \ && case_validate \ From 90f5c8c23e83328527baa91594fe717dc73fd109 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 23 Aug 2023 17:57:24 -0400 Subject: [PATCH 26/29] case_validate: Update NIST inlined license text Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 15 +++++++++------ case_utils/case_validate/validate_types.py | 16 ++++++++++++++++ case_utils/case_validate/validate_utils.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index d1fc70b..4132c01 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/case_validate/validate_types.py b/case_utils/case_validate/validate_types.py index b1911aa..1625963 100644 --- a/case_utils/case_validate/validate_types.py +++ b/case_utils/case_validate/validate_types.py @@ -1,3 +1,19 @@ +#!/usr/bin/env python3 + +# Portions of this file contributed by NIST are governed by the following +# statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. +# +# We would appreciate acknowledgement if the software is used. + from typing import Set, Union import rdflib diff --git a/case_utils/case_validate/validate_utils.py b/case_utils/case_validate/validate_utils.py index e658e05..b7ed317 100644 --- a/case_utils/case_validate/validate_utils.py +++ b/case_utils/case_validate/validate_utils.py @@ -1,3 +1,19 @@ +#!/usr/bin/env python3 + +# Portions of this file contributed by NIST are governed by the following +# statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. +# +# We would appreciate acknowledgement if the software is used. + import importlib import logging import os From 95a52c1fd769c49a6d098f016b4d7d67a1e2f6fb Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Thu, 24 Aug 2023 09:36:42 -0400 Subject: [PATCH 27/29] Update NIST inlined license text Signed-off-by: Alex Nelson --- .github/workflows/cicd.yml | 15 +++++++++------ Makefile | 15 +++++++++------ case_utils/__init__.py | 15 +++++++++------ case_utils/case_file/__init__.py | 15 +++++++++------ case_utils/case_sparql_construct/__init__.py | 15 +++++++++------ case_utils/case_sparql_select/__init__.py | 15 +++++++++------ case_utils/inherent_uuid.py | 15 +++++++++------ case_utils/local_uuid.py | 15 +++++++++------ case_utils/namespace.py | 15 +++++++++------ case_utils/ontology/Makefile | 15 +++++++++------ case_utils/ontology/__init__.py | 15 +++++++++------ .../ontology/src/ontology_and_version_iris.py | 15 +++++++++------ case_utils/ontology/src/subclasses_ttl.py | 15 +++++++++------ case_utils/ontology/version_info.py | 15 +++++++++------ case_utils/py.typed | 15 +++++++++------ setup.py | 15 +++++++++------ tests/Makefile | 15 +++++++++------ tests/case_utils/Makefile | 15 +++++++++------ tests/case_utils/case_file/Makefile | 15 +++++++++------ tests/case_utils/case_file/sample_txt.py | 15 +++++++++------ tests/case_utils/case_file/test_case_file.py | 15 +++++++++------ tests/case_utils/case_sparql_construct/Makefile | 15 +++++++++------ .../test_case_sparql_construct.py | 15 +++++++++------ tests/case_utils/case_sparql_select/Makefile | 15 +++++++++------ .../test_data_frame_to_table_text_json.py | 15 +++++++++------ tests/case_utils/case_validate/Makefile | 15 +++++++++------ .../case_validate/case_test_examples/Makefile | 15 +++++++++------ tests/case_utils/case_validate/cli/Makefile | 15 +++++++++------ .../case_validate/cli/test_format_output_flags.py | 15 +++++++++------ .../case_validate/shape_disabling/Makefile | 15 +++++++++------ .../case_validate/uco_test_examples/Makefile | 15 +++++++++------ tests/case_utils/ontology/test_version_info.py | 15 +++++++++------ tests/case_utils/test_guess_format.py | 15 +++++++++------ tests/case_utils/test_local_uuid.py | 15 +++++++++------ tests/hexbinary/test_hexbinary.py | 15 +++++++++------ tests/isomorphic_diff/Makefile | 15 +++++++++------ tests/src/compact.py | 15 +++++++++------ tests/src/isomorphic_diff.py | 15 +++++++++------ 38 files changed, 342 insertions(+), 228 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 604c1a1..3845a6e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -1,11 +1,14 @@ +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/Makefile b/Makefile index a96ebcd..3d5560f 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/__init__.py b/case_utils/__init__.py index a9456fb..6efe584 100644 --- a/case_utils/__init__.py +++ b/case_utils/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/case_file/__init__.py b/case_utils/case_file/__init__.py index 051e07d..525eea2 100644 --- a/case_utils/case_file/__init__.py +++ b/case_utils/case_file/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/case_sparql_construct/__init__.py b/case_utils/case_sparql_construct/__init__.py index 2ac367f..8bfdf99 100644 --- a/case_utils/case_sparql_construct/__init__.py +++ b/case_utils/case_sparql_construct/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/case_sparql_select/__init__.py b/case_utils/case_sparql_select/__init__.py index be6268d..13d4ca8 100644 --- a/case_utils/case_sparql_select/__init__.py +++ b/case_utils/case_sparql_select/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/inherent_uuid.py b/case_utils/inherent_uuid.py index 667a8d1..8b4026c 100644 --- a/case_utils/inherent_uuid.py +++ b/case_utils/inherent_uuid.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/local_uuid.py b/case_utils/local_uuid.py index 85801b3..5d26522 100644 --- a/case_utils/local_uuid.py +++ b/case_utils/local_uuid.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/namespace.py b/case_utils/namespace.py index 0e02b9c..f1fd4e8 100644 --- a/case_utils/namespace.py +++ b/case_utils/namespace.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/ontology/Makefile b/case_utils/ontology/Makefile index c4cd60d..8c6028c 100644 --- a/case_utils/ontology/Makefile +++ b/case_utils/ontology/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/ontology/__init__.py b/case_utils/ontology/__init__.py index d1430d2..779ffd4 100644 --- a/case_utils/ontology/__init__.py +++ b/case_utils/ontology/__init__.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/ontology/src/ontology_and_version_iris.py b/case_utils/ontology/src/ontology_and_version_iris.py index f0139ed..bc4dd85 100644 --- a/case_utils/ontology/src/ontology_and_version_iris.py +++ b/case_utils/ontology/src/ontology_and_version_iris.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/ontology/src/subclasses_ttl.py b/case_utils/ontology/src/subclasses_ttl.py index f7a25b5..2457852 100644 --- a/case_utils/ontology/src/subclasses_ttl.py +++ b/case_utils/ontology/src/subclasses_ttl.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/ontology/version_info.py b/case_utils/ontology/version_info.py index e603eb7..5ea40f3 100644 --- a/case_utils/ontology/version_info.py +++ b/case_utils/ontology/version_info.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/case_utils/py.typed b/case_utils/py.typed index 3ecd1f5..38c5320 100644 --- a/case_utils/py.typed +++ b/case_utils/py.typed @@ -1,11 +1,14 @@ +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/setup.py b/setup.py index 3a15d74..037c408 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/Makefile b/tests/Makefile index 37ad7e3..f89c724 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/Makefile b/tests/case_utils/Makefile index 3c65a40..0139fc6 100644 --- a/tests/case_utils/Makefile +++ b/tests/case_utils/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_file/Makefile b/tests/case_utils/case_file/Makefile index e2014ec..d65fb40 100644 --- a/tests/case_utils/case_file/Makefile +++ b/tests/case_utils/case_file/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_file/sample_txt.py b/tests/case_utils/case_file/sample_txt.py index 625caa9..d72b57f 100644 --- a/tests/case_utils/case_file/sample_txt.py +++ b/tests/case_utils/case_file/sample_txt.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_file/test_case_file.py b/tests/case_utils/case_file/test_case_file.py index 9194f7d..5aa8673 100644 --- a/tests/case_utils/case_file/test_case_file.py +++ b/tests/case_utils/case_file/test_case_file.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_sparql_construct/Makefile b/tests/case_utils/case_sparql_construct/Makefile index d749fe7..7aa4c9c 100644 --- a/tests/case_utils/case_sparql_construct/Makefile +++ b/tests/case_utils/case_sparql_construct/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_sparql_construct/test_case_sparql_construct.py b/tests/case_utils/case_sparql_construct/test_case_sparql_construct.py index 2c3f37c..dd801d6 100644 --- a/tests/case_utils/case_sparql_construct/test_case_sparql_construct.py +++ b/tests/case_utils/case_sparql_construct/test_case_sparql_construct.py @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_sparql_select/Makefile b/tests/case_utils/case_sparql_select/Makefile index cf13029..153e28a 100644 --- a/tests/case_utils/case_sparql_select/Makefile +++ b/tests/case_utils/case_sparql_select/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_sparql_select/test_data_frame_to_table_text_json.py b/tests/case_utils/case_sparql_select/test_data_frame_to_table_text_json.py index 8fc160d..545666c 100644 --- a/tests/case_utils/case_sparql_select/test_data_frame_to_table_text_json.py +++ b/tests/case_utils/case_sparql_select/test_data_frame_to_table_text_json.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/Makefile b/tests/case_utils/case_validate/Makefile index aa5a489..df9c6ff 100644 --- a/tests/case_utils/case_validate/Makefile +++ b/tests/case_utils/case_validate/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/case_test_examples/Makefile b/tests/case_utils/case_validate/case_test_examples/Makefile index ff33917..c6d4b38 100644 --- a/tests/case_utils/case_validate/case_test_examples/Makefile +++ b/tests/case_utils/case_validate/case_test_examples/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/cli/Makefile b/tests/case_utils/case_validate/cli/Makefile index 5795ad4..b63204e 100644 --- a/tests/case_utils/case_validate/cli/Makefile +++ b/tests/case_utils/case_validate/cli/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/cli/test_format_output_flags.py b/tests/case_utils/case_validate/cli/test_format_output_flags.py index 381c6f3..43d7307 100644 --- a/tests/case_utils/case_validate/cli/test_format_output_flags.py +++ b/tests/case_utils/case_validate/cli/test_format_output_flags.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/shape_disabling/Makefile b/tests/case_utils/case_validate/shape_disabling/Makefile index db3437b..bb10f52 100644 --- a/tests/case_utils/case_validate/shape_disabling/Makefile +++ b/tests/case_utils/case_validate/shape_disabling/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/case_validate/uco_test_examples/Makefile b/tests/case_utils/case_validate/uco_test_examples/Makefile index 921b683..a80f15e 100644 --- a/tests/case_utils/case_validate/uco_test_examples/Makefile +++ b/tests/case_utils/case_validate/uco_test_examples/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/ontology/test_version_info.py b/tests/case_utils/ontology/test_version_info.py index 3f8e6aa..1718d5d 100644 --- a/tests/case_utils/ontology/test_version_info.py +++ b/tests/case_utils/ontology/test_version_info.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/test_guess_format.py b/tests/case_utils/test_guess_format.py index bbfe7d8..5d45cda 100644 --- a/tests/case_utils/test_guess_format.py +++ b/tests/case_utils/test_guess_format.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/case_utils/test_local_uuid.py b/tests/case_utils/test_local_uuid.py index d610cdf..40969fb 100644 --- a/tests/case_utils/test_local_uuid.py +++ b/tests/case_utils/test_local_uuid.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/hexbinary/test_hexbinary.py b/tests/hexbinary/test_hexbinary.py index 7c52fbe..5488c51 100644 --- a/tests/hexbinary/test_hexbinary.py +++ b/tests/hexbinary/test_hexbinary.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/isomorphic_diff/Makefile b/tests/isomorphic_diff/Makefile index 3326a4b..802cda2 100644 --- a/tests/isomorphic_diff/Makefile +++ b/tests/isomorphic_diff/Makefile @@ -1,13 +1,16 @@ #!/usr/bin/make -f +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/src/compact.py b/tests/src/compact.py index e3613a4..95fad38 100644 --- a/tests/src/compact.py +++ b/tests/src/compact.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. diff --git a/tests/src/isomorphic_diff.py b/tests/src/isomorphic_diff.py index 5caa698..445e621 100644 --- a/tests/src/isomorphic_diff.py +++ b/tests/src/isomorphic_diff.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 +# Portions of this file contributed by NIST are governed by the following +# statement: +# # This software was developed at the National Institute of Standards # and Technology by employees of the Federal Government in the course -# of their official duties. Pursuant to title 17 Section 105 of the -# United States Code this software is not subject to copyright -# protection and is in the public domain. NIST assumes no -# responsibility whatsoever for its use by other parties, and makes -# no guarantees, expressed or implied, about its quality, -# reliability, or any other characteristic. +# of their official duties. Pursuant to Title 17 Section 105 of the +# United States Code, this software is not subject to copyright +# protection within the United States. NIST assumes no responsibility +# whatsoever for its use by other parties, and makes no guarantees, +# expressed or implied, about its quality, reliability, or any other +# characteristic. # # We would appreciate acknowledgement if the software is used. From 41863631d05bccc0cd3d879ec672ec262165d640 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Thu, 24 Aug 2023 16:17:32 -0400 Subject: [PATCH 28/29] (Behavior change) Add --review-tbox flag This is part of addressing what originally inspired pySHACL Issue 170. This patch modifies the behavior of `case_validate`, that reviews OWL syntax and OWL-SHACL interactions. With this patch, that functionality is now **opt-in** at call time. Further work on separating the OWL review shapes from UCO into a general CDO repository (originally started for CDOTSC-34) is currently believed to not have an impact on the user interface element where the user opts in to the more extensive review. References: * [CDOTSC-34] CDO should provide shapes for external ontologies and other RDF models, including OWL * https://github.com/RDFLib/pySHACL/issues/170 Signed-off-by: Alex Nelson --- case_utils/case_validate/__init__.py | 51 ++++++++----------- case_utils/case_validate/validate_utils.py | 19 +++++++ .../case_validate/uco_test_examples/Makefile | 31 +++++++++++ 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index a214cf9..22de866 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -50,6 +50,7 @@ ValidationResult, ) from case_utils.case_validate.validate_utils import ( + disable_tbox_review, get_invalid_cdo_concepts, get_ontology_graph, ) @@ -65,6 +66,7 @@ def validate( input_file: Union[List[str], str], *args: Any, case_version: Optional[str] = None, + review_tbox: bool = False, supplemental_graphs: Optional[List[str]] = None, **kwargs: Any, ) -> ValidationResult: @@ -74,6 +76,7 @@ def validate( :param *args: The positional arguments to pass to the underlying pyshacl.validate function. :param input_file: The path to the file containing the data graph to validate. This can also be a list of paths to files containing data graphs to pool together. :param case_version: The version of the CASE ontology to use (e.g. 1.2.0). If None, the most recent version will be used. + :param review_tbox: If True, SHACL shapes that review OWL Classes, OWL Properties, and SHACL shapes that constrain those classes and properties will be used in the review. Otherwise, those shapes will be deactivated before running validation. Be aware that these shapes are known to significantly increase the validation run time. :param supplemental_graphs: File paths to supplemental graphs to use. If None, no supplemental graphs will be used. :param allow_warnings: In addition to affecting the conformance of SHACL validation, this will affect conformance based on unrecognized CDO concepts (likely, misspelled or miscapitalized) in the data graph. If allow_warnings is not True, any unrecognized concept using a CDO IRI prefix will cause conformance to be False. :param inference: The type of inference to use. If "none" (type str), no inference will be used. If None (type NoneType), pyshacl defaults will be used. Note that at the time of this writing (pySHACL 0.23.0), pyshacl defaults are no inferencing for the data graph, and RDFS inferencing for the SHACL graph, which for case_utils.validate includes the SHACL and OWL graphs. @@ -94,36 +97,16 @@ def validate( # Get the ontology graph from the case_version and supplemental_graphs arguments ontology_graph: Graph = get_ontology_graph(case_version, supplemental_graphs) - # Filter the graph pyshacl uses as its ontology mix-in to exclude - # all SHACL-related triples. - # This is done because, at the time of pyshacl 0.20.0, the entirety - # of the ontology graph is mixed into the data graph. UCO 1.0.0 - # includes some mechanisms to cross-check SHACL PropertyShapes - # versus OWL property definitions. Because of the mix-in, all of - # the ontology graph (.validate ont_graph kwarg) is reviewed by the - # SHACL graph (.validate shacl_graph kwarg), so for UCO 1.0.0 that - # adds around 30 seconds to each case_validate call, redundantly - # reviewing UCO. - # The ontology graph (.validate ont_graph kwarg) is currently - # believed to never need to know about SHACL concepts. - ontology_graph_without_shacl = rdflib.Graph() - SH_prefix = str(rdflib.SH) - for triple in ontology_graph.triples((None, None, None)): - skip_triple = False - for triple_part in triple: - if isinstance(triple_part, rdflib.URIRef): - if str(triple_part).startswith(SH_prefix): - skip_triple = True - if skip_triple: - break - if skip_triple: - continue - ontology_graph_without_shacl.add(triple) - # _logger.debug("len(ontology_graph) = %d.", len(ontology_graph)) - # _logger.debug("len(ontology_graph_without_shacl) = %d.", len(ontology_graph_without_shacl)) - # At the time of CASE 1.0.0, this was the debug output: - # DEBUG:__init__.py:len(ontology_graph) = 13499. - # DEBUG:__init__.py:len(ontology_graph_without_shacl) = 7639. + if not review_tbox: + # This is done because, at the time of pyshacl 0.20.0, the + # entirety of the ontology graph is mixed into the data graph. + # UCO 1.0.0 includes some mechanisms to cross-check SHACL + # PropertyShapes versus OWL property definitions. Because of + # the mix-in, all of the ontology graph (.validate ont_graph + # kwarg) is reviewed by the SHACL graph (.validate shacl_graph + # kwarg), so for UCO 1.0.0 that adds around 30 seconds to each + # case_validate call, redundantly reviewing UCO. + disable_tbox_review(ontology_graph) # Get the undefined CDO concepts. undefined_cdo_concepts = get_invalid_cdo_concepts(data_graph, ontology_graph) @@ -142,7 +125,7 @@ def validate( ] = pyshacl.validate( data_graph, *args, - ont_graph=ontology_graph_without_shacl, + ont_graph=ontology_graph, shacl_graph=ontology_graph, **kwargs, ) @@ -256,6 +239,11 @@ def main() -> None: help='(ALMOST as with pyshacl CLI) Send output to a file. If absent, output will be written to stdout. Difference: If specified, file is expected not to exist. Clarification: Does NOT influence --format flag\'s default value of "human". (I.e., any machine-readable serialization format must be specified with --format.)', default=sys.stdout, ) + parser.add_argument( + "--review-tbox", + action="store_true", + help='Enable rules for reviewing OWL Classes, Properties, and SHACL shapes that constrain them (i.e. the "TBox", or "Theorem box", of the data graph and ontology graph; in contrast, the "ABox", or "Axiom box", contains the declarations of members of those classes, and users of those properties). This should be used when adding extension classes or properties not adopted by UCO or its downstream ontologies, e.g. when using a drafting namespace. Be aware that these rules are known to significantly increase the validation run time.', + ) parser.add_argument("in_graph", nargs="+") @@ -281,6 +269,7 @@ def main() -> None: do_owl_imports=True if args.imports else False, inference=args.inference, meta_shacl=args.metashacl, + review_tbox=True if args.review_tbox else False, supplemental_graphs=args.ontology_graph, **validator_kwargs, ) diff --git a/case_utils/case_validate/validate_utils.py b/case_utils/case_validate/validate_utils.py index b7ed317..44c1e93 100644 --- a/case_utils/case_validate/validate_utils.py +++ b/case_utils/case_validate/validate_utils.py @@ -182,3 +182,22 @@ def get_ontology_graph( ontology_graph.parse(arg_ontology_graph) return ontology_graph + + +def disable_tbox_review(graph: rdflib.Graph) -> None: + l_true = rdflib.Literal(True) + ns_uco_owl = rdflib.Namespace("https://ontology.unifiedcyberontology.org/owl/") + + for tbox_shape_basename in { + "DataOneOf-shape", + "DatatypeProperty-shacl-constraints-shape", + "Disjointedness-AP-DP-shape", + "Disjointedness-AP-OP-shape", + "Disjointedness-C-DT-shape", + "Disjointedness-DP-OP-shape", + "ObjectProperty-shacl-constraints-shape", + "ontologyIRI-versionIRI-prerequisite-shape", + "versionIRI-nodeKind-shape", + }: + n_tbox_shape = ns_uco_owl[tbox_shape_basename] + graph.add((n_tbox_shape, NS_SH.deactivated, l_true)) diff --git a/tests/case_utils/case_validate/uco_test_examples/Makefile b/tests/case_utils/case_validate/uco_test_examples/Makefile index 921b683..07b75c9 100644 --- a/tests/case_utils/case_validate/uco_test_examples/Makefile +++ b/tests/case_utils/case_validate/uco_test_examples/Makefile @@ -99,6 +99,37 @@ all: \ rm __$@ mv _$@ $@ +# NOTE - this more-specific recipe enables "tbox" review, but otherwise +# matches the wildcarded recipe. +owl_properties_XFAIL_validation.ttl: \ + $(examples_srcdir)/owl_properties_XFAIL.json \ + $(tests_srcdir)/.venv.done.log \ + $(top_srcdir)/.ontology.done.log \ + $(top_srcdir)/case_utils/case_validate/__init__.py \ + $(top_srcdir)/case_utils/case_validate/validate_types.py \ + $(top_srcdir)/case_utils/case_validate/validate_utils.py \ + $(top_srcdir)/case_utils/ontology/__init__.py + source $(tests_srcdir)/venv/bin/activate \ + && case_validate \ + --allow-warnings \ + --debug \ + --format turtle \ + --review-tbox \ + $< \ + > __$@ \ + ; rc=$$? ; test 0 -eq $$rc -o 1 -eq $$rc + @#Fail if output is empty. + @test -s __$@ \ + || exit 1 + java -jar $(RDF_TOOLKIT_JAR) \ + --inline-blank-nodes \ + --source __$@ \ + --source-format turtle \ + --target _$@ \ + --target-format turtle + rm __$@ + mv _$@ $@ + check: \ $(validation_ttls) source $(tests_srcdir)/venv/bin/activate \ From 0e895b63ab6491c727dd8e2539d83eee4abf813e Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Fri, 25 Aug 2023 10:57:57 -0400 Subject: [PATCH 29/29] Bump versions Signed-off-by: Alex Nelson --- case_utils/__init__.py | 2 +- case_utils/case_file/__init__.py | 2 +- case_utils/case_sparql_construct/__init__.py | 2 +- case_utils/case_sparql_select/__init__.py | 2 +- case_utils/case_validate/__init__.py | 2 +- case_utils/case_validate/validate_types.py | 2 ++ case_utils/case_validate/validate_utils.py | 2 ++ case_utils/inherent_uuid.py | 2 +- case_utils/local_uuid.py | 2 +- case_utils/namespace.py | 2 +- case_utils/ontology/__init__.py | 2 +- case_utils/ontology/src/ontology_and_version_iris.py | 2 +- case_utils/ontology/src/subclasses_ttl.py | 2 +- case_utils/ontology/version_info.py | 2 +- tests/src/compact.py | 2 +- tests/src/isomorphic_diff.py | 2 +- 16 files changed, 18 insertions(+), 14 deletions(-) diff --git a/case_utils/__init__.py b/case_utils/__init__.py index 6efe584..c98638f 100644 --- a/case_utils/__init__.py +++ b/case_utils/__init__.py @@ -14,6 +14,6 @@ # # We would appreciate acknowledgement if the software is used. -__version__ = "0.11.0" +__version__ = "0.12.0" from . import local_uuid # noqa: F401 diff --git a/case_utils/case_file/__init__.py b/case_utils/case_file/__init__.py index 525eea2..5f797fc 100644 --- a/case_utils/case_file/__init__.py +++ b/case_utils/case_file/__init__.py @@ -18,7 +18,7 @@ This module creates a graph object that provides a basic UCO characterization of a single file. The gathered metadata is among the more "durable" file characteristics, i.e. characteristics that would remain consistent when transferring a file between locations. """ -__version__ = "0.5.0" +__version__ = "0.5.1" import argparse import datetime diff --git a/case_utils/case_sparql_construct/__init__.py b/case_utils/case_sparql_construct/__init__.py index 8bfdf99..46072c0 100644 --- a/case_utils/case_sparql_construct/__init__.py +++ b/case_utils/case_sparql_construct/__init__.py @@ -18,7 +18,7 @@ This script executes a SPARQL CONSTRUCT query, returning a graph of the generated triples. """ -__version__ = "0.2.5" +__version__ = "0.2.6" import argparse import logging diff --git a/case_utils/case_sparql_select/__init__.py b/case_utils/case_sparql_select/__init__.py index 13d4ca8..f1c7bb7 100644 --- a/case_utils/case_sparql_select/__init__.py +++ b/case_utils/case_sparql_select/__init__.py @@ -29,7 +29,7 @@ Should a more complex query be necessary, an outer, wrapping SELECT query would let this script continue to function. """ -__version__ = "0.5.1" +__version__ = "0.5.2" import argparse import binascii diff --git a/case_utils/case_validate/__init__.py b/case_utils/case_validate/__init__.py index 22de866..babea69 100644 --- a/case_utils/case_validate/__init__.py +++ b/case_utils/case_validate/__init__.py @@ -32,7 +32,7 @@ details.) """ -__version__ = "0.3.0" +__version__ = "0.4.0" import argparse import logging diff --git a/case_utils/case_validate/validate_types.py b/case_utils/case_validate/validate_types.py index 1625963..8ca825f 100644 --- a/case_utils/case_validate/validate_types.py +++ b/case_utils/case_validate/validate_types.py @@ -14,6 +14,8 @@ # # We would appreciate acknowledgement if the software is used. +__version__ = "0.1.0" + from typing import Set, Union import rdflib diff --git a/case_utils/case_validate/validate_utils.py b/case_utils/case_validate/validate_utils.py index 44c1e93..4f711e0 100644 --- a/case_utils/case_validate/validate_utils.py +++ b/case_utils/case_validate/validate_utils.py @@ -14,6 +14,8 @@ # # We would appreciate acknowledgement if the software is used. +__version__ = "0.1.0" + import importlib import logging import os diff --git a/case_utils/inherent_uuid.py b/case_utils/inherent_uuid.py index 8b4026c..cdcba73 100644 --- a/case_utils/inherent_uuid.py +++ b/case_utils/inherent_uuid.py @@ -57,7 +57,7 @@ >>> assert str(n_file_facet)[-36:] == str(n_file_facet_2)[-36:] """ -__version__ = "0.1.0" +__version__ = "0.1.1" import binascii import re diff --git a/case_utils/local_uuid.py b/case_utils/local_uuid.py index 2691daa..2df5fcb 100644 --- a/case_utils/local_uuid.py +++ b/case_utils/local_uuid.py @@ -20,7 +20,7 @@ The function local_uuid() should be used in code where a user could be expected to opt in to non-random UUIDs. """ -__version__ = "0.3.2" +__version__ = "0.4.0" __all__ = ["configure", "local_uuid"] diff --git a/case_utils/namespace.py b/case_utils/namespace.py index f1fd4e8..62e9475 100644 --- a/case_utils/namespace.py +++ b/case_utils/namespace.py @@ -20,7 +20,7 @@ To use, add "from case_utils.namespace import *". Namespace variables starting with "NS_" are imported. As needs are demonstrated in CASE tooling (both in case_utils and from downstream requests), namespaces will also be imported from rdflib for a consistent "NS_*" spelling. """ -__version__ = "0.2.1" +__version__ = "0.2.2" import rdflib diff --git a/case_utils/ontology/__init__.py b/case_utils/ontology/__init__.py index 779ffd4..0eafc5c 100644 --- a/case_utils/ontology/__init__.py +++ b/case_utils/ontology/__init__.py @@ -14,7 +14,7 @@ # # We would appreciate acknowledgement if the software is used. -__version__ = "0.1.1" +__version__ = "0.1.2" import importlib.resources import logging diff --git a/case_utils/ontology/src/ontology_and_version_iris.py b/case_utils/ontology/src/ontology_and_version_iris.py index bc4dd85..9b86deb 100644 --- a/case_utils/ontology/src/ontology_and_version_iris.py +++ b/case_utils/ontology/src/ontology_and_version_iris.py @@ -18,7 +18,7 @@ This script creates a list of all ontology and version IRIs that have ever existed in a CDO ontology to describe a CDO ontology. I.e. the subject of triples with owl:Ontology as predicate are included, as are the objects of version-referencing triples (owl:versionIRI, owl:incompatibleWith, etc.). """ -__version__ = "0.1.1" +__version__ = "0.1.2" import argparse import typing diff --git a/case_utils/ontology/src/subclasses_ttl.py b/case_utils/ontology/src/subclasses_ttl.py index 2457852..1bc03ca 100644 --- a/case_utils/ontology/src/subclasses_ttl.py +++ b/case_utils/ontology/src/subclasses_ttl.py @@ -18,7 +18,7 @@ This script creates an excerpt of an ontology graph that consists solely of all rdfs:subClassOf statements. """ -__version__ = "0.1.0" +__version__ = "0.1.1" import argparse diff --git a/case_utils/ontology/version_info.py b/case_utils/ontology/version_info.py index 5ea40f3..7bdf9a3 100644 --- a/case_utils/ontology/version_info.py +++ b/case_utils/ontology/version_info.py @@ -22,7 +22,7 @@ When preparing to build a new monolithic ontology, please edit this variable to match the new respective version. """ -__version__ = "0.6.0" +__version__ = "0.6.1" __all__ = ["CURRENT_CASE_VERSION", "built_version_choices_list"] diff --git a/tests/src/compact.py b/tests/src/compact.py index 95fad38..96c2828 100644 --- a/tests/src/compact.py +++ b/tests/src/compact.py @@ -21,7 +21,7 @@ https://www.w3.org/TR/json-ld11/#terms """ -__version__ = "0.2.1" +__version__ = "0.2.2" import json import logging diff --git a/tests/src/isomorphic_diff.py b/tests/src/isomorphic_diff.py index 445e621..b2f3b7d 100644 --- a/tests/src/isomorphic_diff.py +++ b/tests/src/isomorphic_diff.py @@ -30,7 +30,7 @@ On resolution of rdflib issue 812, and on adding some format-support flexibility, isomorphic_diff.py is likely to be deprecated in favor of using the upstream rdfgraphisomorphism command. """ -__version__ = "0.2.1" +__version__ = "0.2.2" import argparse import logging