Include instances as attributes of the relevant class

Author: apdavison

build.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+import json
 import os.path
 import shutil
 import subprocess
@@ -7,25 +8,35 @@
 from jinja2 import Environment, select_autoescape, FileSystemLoader
 
 from pipeline.translator import PythonBuilder
-from pipeline.utils import clone_sources, SchemaLoader
+from pipeline.utils import clone_sources, SchemaLoader, InstanceLoader
 
-print("***************************************")
+print("*********************************************************")
 print(f"Triggering the generation of Python package for openMINDS")
-print("***************************************")
+print("*********************************************************")
 
 # Step 1 - clone central repository in main branch to get the latest sources
 clone_sources()
 schema_loader = SchemaLoader()
+instance_loader = InstanceLoader()
 if os.path.exists("target"):
     shutil.rmtree("target")
 
+# Step 2 - load instances
+instances = {}
+for version in instance_loader.get_instance_versions():
+    instances[version] = defaultdict(list)
+    for instance_path in instance_loader.find_instances(version):
+        with open(instance_path) as fp:
+            instance_data = json.load(fp)
+        instances[version][instance_data["@type"]].append(instance_data)
+
 python_modules = defaultdict(list)
 for schema_version in schema_loader.get_schema_versions():
 
-    # Step 2 - find all involved schemas for the current version
+    # Step 3 - find all involved schemas for the current version
     schemas_file_paths = schema_loader.find_schemas(schema_version)
 
-    # Step 3a - figure out which schemas are embedded and which are linked
+    # Step 4a - figure out which schemas are embedded and which are linked
     embedded = set()
     linked = set()
     for schema_file_path in schemas_file_paths:
@@ -42,17 +53,17 @@
         for schema_identifier in conflicts:
             linked.remove(schema_identifier)
 
-    # Step 3b - translate and build each openMINDS schema as JSON-Schema
+    # Step 4b - translate and build each openMINDS schema as JSON-Schema
     for schema_file_path in schemas_file_paths:
         module_path, class_name = PythonBuilder(
-            schema_file_path, schema_loader.schemas_sources
+            schema_file_path, schema_loader.schemas_sources, instances=instances.get(schema_version, None)
         ).build(embedded=embedded)
 
         parts = module_path.split(".")
         parent_path = ".".join(parts[:-1])
         python_modules[parent_path].append((parts[-1], class_name))
 
-# Step 4 - create additional files, e.g. __init__.py
+# Step 5 - create additional files, e.g. __init__.py
 openminds_modules = defaultdict(set)
 for path, classes in python_modules.items():
     dir_path = ["target", "openminds"] + path.split(".")
@@ -96,5 +107,5 @@
 shutil.copy("pipeline/src/README.md", "target/README.md")
 shutil.copy("./LICENSE", "target/LICENSE")
 
-# Step 5 - run formatter
+# Step 6 - run formatter
 subprocess.call([sys.executable, "-m", "black", "--quiet", "target"])

pipeline/src/module_template.py.txt

@@ -47,3 +47,10 @@ class {{ class_name }}({{ base_class }}):
         return super().__init__({%- if base_class == "LinkedMetadata" %}id=id, {%- endif -%}{%- for property in properties -%}{{property.name}}={{property.name}}, {%- endfor -%})
 
 {{ additional_methods }}
+
+{% for instance_name, instance in instances.items() %}
+{{ class_name }}.{{ instance_name }} = {{ class_name }}(
+    {% for key, value in instance.items() -%}
+    {% if value is string %}{{key}}="{{value}}",{% else %}{{key}}={{value}},{% endif %}
+    {% endfor -%}
+){% endfor %}

pipeline/translator.py

@@ -6,10 +6,27 @@
 from jinja2 import Environment, select_autoescape, FileSystemLoader
 
 
+number_names = {
+    "0": "zero",
+    "1": "one",
+    "2": "two",
+    "3": "three",
+    "4": "four",
+    "5": "five",
+    "6": "six",
+    "7": "seven",
+    "8": "eight",
+    "9": "nine"
+}
+
+
 def generate_python_name(json_name, allow_multiple=False):
     python_name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", json_name)
     python_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", python_name).lower()
-    python_name = python_name.replace("-", "_")
+    python_name = python_name.replace("-", "_").replace(".", "_").replace("+", "plus").replace("#", "sharp")
+    if python_name[0] in number_names:  # Python variables can't start with a number
+        python_name = number_names[python_name[0]] + python_name[1:]
+
     # if (
     #     allow_multiple
     #     and python_name[-1] != "s"
@@ -32,7 +49,7 @@ def generate_python_name(json_name, allow_multiple=False):
 class PythonBuilder(object):
     """docstring"""
 
-    def __init__(self, schema_file_path: str, root_path: str):
+    def __init__(self, schema_file_path: str, root_path: str, instances: Optional[dict] = None):
         self.template_name = "src/module_template.py.txt"
         self.env = Environment(
             loader=FileSystemLoader(os.path.dirname(os.path.realpath(__file__))), autoescape=select_autoescape()
@@ -46,6 +63,7 @@ def __init__(self, schema_file_path: str, root_path: str):
         ]
         with open(schema_file_path, "r") as schema_f:
             self._schema_payload = json.load(schema_f)
+        self.instances = instances or {}
 
     @property
     def _version_module(self):
@@ -106,16 +124,40 @@ def get_type(property):
             else:
                 raise NotImplementedError
 
-        if self._schema_payload["_type"] in embedded:
+        openminds_type = self._schema_payload["_type"]
+        if openminds_type in embedded:
             base_class = "EmbeddedMetadata"
         else:
             base_class = "LinkedMetadata"
+
+        def filter_value(value):
+            if isinstance(value, str):
+                return value.replace('"', "'").replace("\n", " ")
+            return value
+
+        def filter_instance(instance):
+            filtered_instance = {
+                generate_python_name(k): filter_value(v)
+                for k, v in instance.items()
+                if k[0] != "@" and k[:4] != "http" and v is not None
+            }
+            filtered_instance["id"] = instance["@id"]
+            return filtered_instance
+
+        instances = {
+            generate_python_name(instance["@id"].split("/")[-1]) : filter_instance(instance)
+            for instance in self.instances.get(openminds_type, [])
+        }
+        instances = {  # sort by key
+            name: instances[name] for name in sorted(instances)
+        }
+
         self.context = {
             "docstring": self._schema_payload.get("description", "<description not available>"),
             "base_class": base_class,
             "preamble": "",  # todo: e.g. extra imports
             "class_name": self._schema_payload["name"],
-            "openminds_type": self._schema_payload["_type"],
+            "openminds_type": openminds_type,
             "schema_version": self.version,
             "properties": [  # call this "properties"
                 {
@@ -136,6 +178,7 @@ def get_type(property):
             ],  # todo
             # unused in property:  "nameForReverseLink"
             "additional_methods": "",
+            "instances": instances
         }
         import_map = {
             "date": "from datetime import date",

pipeline/utils.py

@@ -5,22 +5,51 @@
 
 from git import Repo, GitCommandError
 
-source_url = "https://github.com/openMetadataInitiative/openMINDS.git"
-
 
 def clone_sources():
-    if os.path.exists("sources"):
-        shutil.rmtree("sources")
-    Repo.clone_from(source_url, to_path="sources", depth=1)
-
-
-class SchemaLoader(object):
+    if os.path.exists("_sources"):
+        shutil.rmtree("_sources")
+    Repo.clone_from(
+        "https://github.com/openMetadataInitiative/openMINDS.git",
+        to_path="_sources",
+        depth=1,
+    )
+    if os.path.exists("_instances"):
+        shutil.rmtree("_instances")
+    Repo.clone_from(
+        "https://github.com/openMetadataInitiative/openMINDS_instances.git",
+        to_path="_instances",
+        depth=1,
+    )
+
+
+class SchemaLoader:
     def __init__(self):
         self._root_directory = os.path.realpath(".")
-        self.schemas_sources = os.path.join(self._root_directory, "sources", "schemas")
+        self.schemas_sources = os.path.join(self._root_directory, "_sources", "schemas")
 
     def get_schema_versions(self) -> List[str]:
         return os.listdir(self.schemas_sources)
 
     def find_schemas(self, version: str) -> List[str]:
-        return glob.glob(os.path.join(self.schemas_sources, version, f"**/*.schema.omi.json"), recursive=True)
+        return glob.glob(
+            os.path.join(self.schemas_sources, version, f"**/*.schema.omi.json"),
+            recursive=True,
+        )
+
+
+class InstanceLoader:
+    def __init__(self):
+        self._root_directory = os.path.realpath(".")
+        self.instances_sources = os.path.join(
+            self._root_directory, "_instances", "instances"
+        )
+
+    def get_instance_versions(self) -> List[str]:
+        return os.listdir(self.instances_sources)
+
+    def find_instances(self, version: str) -> List[str]:
+        return glob.glob(
+            os.path.join(self.instances_sources, version, f"**/*.jsonld"),
+            recursive=True,
+        )