diff --git a/docs/source/netcdf.rst b/docs/source/netcdf.rst index 2ff50c4..fb85ea2 100644 --- a/docs/source/netcdf.rst +++ b/docs/source/netcdf.rst @@ -11,7 +11,7 @@ IMAS netCDF files IMAS-Python supports reading IDSs from and writing IDSs to IMAS netCDF files. This feature is currently in alpha status, and its functionality may change in -upcoming minor releases of IMAS-Python. +upcoming (minor) releases of IMAS-Python. A detailed description of the IMAS netCDF format and conventions can be found on the :ref:`IMAS conventions for the netCDF data format` page. @@ -42,6 +42,34 @@ will be used for :py:meth:`~imas.db_entry.DBEntry.get` and imas.util.print_tree(cp2) +Implemented features of a netCDF ``DBEntry`` +-------------------------------------------- + +A netCDF ``DBEntry`` doesn't implement all features that are supported by +``imas_core``. The following table provides an overview of the implemented +features that are supported by DBEntries using ``imas_core`` respectively +``netCDF``: + +.. list-table:: + :header-rows: 1 + + * - Feature + - ``imas_core`` + - ``netCDF`` + * - :ref:`Lazy loading` + - Yes + - Yes + * - :ref:`Automatic conversion between DD versions ` + - When reading and writing + - When reading + * - ``get_slice`` / ``put_slice`` + - Yes + - Not implemented + * - ``get_sample`` + - Yes (requires ``imas_core >= 5.4.0``) + - Not implemented + + Using IMAS netCDF files with 3rd-party tools -------------------------------------------- diff --git a/imas/backends/imas_core/al_context.py b/imas/backends/imas_core/al_context.py index b33c99b..19b34d8 100644 --- a/imas/backends/imas_core/al_context.py +++ b/imas/backends/imas_core/al_context.py @@ -10,6 +10,7 @@ import numpy +import imas from imas.backends.imas_core.imas_interface import ll_interface from imas.exception import LowlevelError from imas.ids_defs import ( @@ -280,6 +281,9 @@ def __init__( self.context = None """Potential weak reference to opened context.""" + def get_child(self, child): + imas.backends.imas_core.db_entry_helpers._get_child(child, self) + def get_context(self) -> ALContext: """Create and yield the actual ALContext.""" if self.dbentry._db_ctx is not self.dbentry_ctx: diff --git a/imas/backends/imas_core/db_entry_helpers.py b/imas/backends/imas_core/db_entry_helpers.py index 4216db5..f83a0d4 100644 --- a/imas/backends/imas_core/db_entry_helpers.py +++ b/imas/backends/imas_core/db_entry_helpers.py @@ -22,7 +22,7 @@ def get_children( structure: IDSStructure, ctx: ALContext, time_mode: int, - nbc_map: Optional[NBCPathMap], + nbc_map: Optional["NBCPathMap"], ) -> None: """Recursively get all children of an IDSStructure.""" # NOTE: changes in this method must be propagated to _get_child and vice versa @@ -77,15 +77,11 @@ def get_children( getattr(structure, name)._IDSPrimitive__value = data -def _get_child(child: IDSBase, ctx: Optional[LazyALContext]): +def _get_child(child: IDSBase, ctx: LazyALContext): """Get a single child when required (lazy loading).""" # NOTE: changes in this method must be propagated to _get_children and vice versa # Performance: this method is specialized for the lazy get - # ctx can be None when the parent structure does not exist in the on-disk DD version - if ctx is None: - return # There is no data to be loaded - time_mode = ctx.time_mode if time_mode == IDS_TIME_MODE_INDEPENDENT and child.metadata.type.is_dynamic: return # skip dynamic (time-dependent) nodes @@ -148,7 +144,7 @@ def put_children( ctx: ALContext, time_mode: int, is_slice: bool, - nbc_map: Optional[NBCPathMap], + nbc_map: Optional["NBCPathMap"], verify_maxoccur: bool, ) -> None: """Recursively put all children of an IDSStructure""" diff --git a/imas/backends/imas_core/imas_interface.py b/imas/backends/imas_core/imas_interface.py index 6f4b3ba..6e46330 100644 --- a/imas/backends/imas_core/imas_interface.py +++ b/imas/backends/imas_core/imas_interface.py @@ -32,7 +32,7 @@ imasdef = None lowlevel = None logger.critical( - "Could not import 'al_core': %s. Some functionality is not available.", + "Could not import 'imas_core': %s. Some functionality is not available.", exc, ) diff --git a/imas/backends/netcdf/db_entry_nc.py b/imas/backends/netcdf/db_entry_nc.py index a23005a..e6ee32c 100644 --- a/imas/backends/netcdf/db_entry_nc.py +++ b/imas/backends/netcdf/db_entry_nc.py @@ -11,7 +11,7 @@ from imas.backends.netcdf.ids2nc import IDS2NC from imas.backends.netcdf.nc2ids import NC2IDS from imas.exception import DataEntryException, InvalidNetCDFEntry -from imas.ids_convert import NBCPathMap, convert_ids +from imas.ids_convert import NBCPathMap, dd_version_map_from_factories from imas.ids_factory import IDSFactory from imas.ids_toplevel import IDSToplevel @@ -108,10 +108,6 @@ def get( else: func = "get_sample" raise NotImplementedError(f"`{func}` is not available for netCDF files.") - if lazy: - raise NotImplementedError( - "Lazy loading is not implemented for netCDF files." - ) # Check if the IDS/occurrence exists, and obtain the group it is stored in try: @@ -123,14 +119,19 @@ def get( # Load data into the destination IDS if self._ds_factory.dd_version == destination._dd_version: - NC2IDS(group, destination).run() + NC2IDS(group, destination, destination.metadata, None).run(lazy) else: - # FIXME: implement automatic conversion using nbc_map - # As a work-around: do an explicit conversion, but automatic conversion - # will also be needed to implement lazy loading. - ids = self._ds_factory.new(ids_name) - NC2IDS(group, ids).run() - convert_ids(ids, None, target=destination) + # Construct relevant NBCPathMap, the one we get from DBEntry has the reverse + # mapping from what we need. The imas_core logic does the mapping from + # in-memory to on-disk, while we take what is on-disk and map it to + # in-memory. + ddmap, source_is_older = dd_version_map_from_factories( + ids_name, self._ds_factory, self._factory + ) + nbc_map = ddmap.old_to_new if source_is_older else ddmap.new_to_old + NC2IDS( + group, destination, self._ds_factory.new(ids_name).metadata, nbc_map + ).run(lazy) return destination diff --git a/imas/backends/netcdf/nc2ids.py b/imas/backends/netcdf/nc2ids.py index 50668df..7829d25 100644 --- a/imas/backends/netcdf/nc2ids.py +++ b/imas/backends/netcdf/nc2ids.py @@ -3,11 +3,13 @@ from typing import Iterator, List, Optional, Tuple import netCDF4 +import numpy as np from imas.backends.netcdf import ids2nc from imas.backends.netcdf.nc_metadata import NCMetadata from imas.exception import InvalidNetCDFEntry from imas.ids_base import IDSBase +from imas.ids_convert import NBCPathMap from imas.ids_data_type import IDSDataType from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS from imas.ids_metadata import IDSMetadata @@ -70,22 +72,37 @@ def _tree_iter( class NC2IDS: """Class responsible for reading an IDS from a NetCDF group.""" - def __init__(self, group: netCDF4.Group, ids: IDSToplevel) -> None: + def __init__( + self, + group: netCDF4.Group, + ids: IDSToplevel, + ids_metadata: IDSMetadata, + nbc_map: Optional[NBCPathMap], + ) -> None: """Initialize NC2IDS converter. Args: group: NetCDF group that stores the IDS data. ids: Corresponding IDS toplevel to store the data in. + ids_metadata: Metadata corresponding to the DD version that the data is + stored in. + nbc_map: Path map for implicit DD conversions. """ self.group = group """NetCDF Group that the IDS is stored in.""" self.ids = ids """IDS to store the data in.""" + self.ids_metadata = ids_metadata + """Metadata of the IDS in the DD version that the data is stored in""" + self.nbc_map = nbc_map + """Path map for implicit DD conversions.""" - self.ncmeta = NCMetadata(ids.metadata) + self.ncmeta = NCMetadata(ids_metadata) """NetCDF related metadata.""" self.variables = list(group.variables) """List of variable names stored in the netCDF group.""" + + self._lazy_map = {} # Don't use masked arrays: they're slow and we'll handle most of the unset # values through the `:shape` arrays self.group.set_auto_mask(False) @@ -99,7 +116,7 @@ def __init__(self, group: netCDF4.Group, ids: IDSToplevel) -> None: "Mandatory variable `ids_properties.homogeneous_time` does not exist." ) var = group["ids_properties.homogeneous_time"] - self._validate_variable(var, ids.ids_properties.homogeneous_time.metadata) + self._validate_variable(var, ids.metadata["ids_properties/homogeneous_time"]) if var[()] not in [0, 1, 2]: raise InvalidNetCDFEntry( f"Invalid value for ids_properties.homogeneous_time: {var[()]}. " @@ -107,23 +124,52 @@ def __init__(self, group: netCDF4.Group, ids: IDSToplevel) -> None: ) self.homogeneous_time = var[()] == IDS_TIME_MODE_HOMOGENEOUS - def run(self) -> None: + def run(self, lazy: bool) -> None: """Load the data from the netCDF group into the IDS.""" self.variables.sort() self.validate_variables() + if lazy: + self.ids._set_lazy_context(LazyContext(self)) for var_name in self.variables: if var_name.endswith(":shape"): continue - metadata = self.ids.metadata[var_name] + metadata = self.ids_metadata[var_name] if metadata.data_type is IDSDataType.STRUCTURE: continue # This only contains DD metadata we already know + # Handle implicit DD version conversion + if self.nbc_map is None: + target_metadata = metadata # no conversion + elif metadata.path_string in self.nbc_map: + new_path = self.nbc_map.path[metadata.path_string] + if new_path is None: + logging.info( + "Not loading data for %s: no equivalent data structure exists " + "in the target Data Dictionary version.", + metadata.path_string, + ) + continue + target_metadata = self.ids.metadata[new_path] + elif metadata.path_string in self.nbc_map.type_change: + logging.info( + "Not loading data for %s: cannot hanlde type changes when " + "implicitly converting data to the target Data Dictionary version.", + metadata.path_string, + ) + continue + else: + target_metadata = metadata # no conversion required + var = self.group[var_name] + if lazy: + self._lazy_map[target_metadata.path_string] = var + continue + if metadata.data_type is IDSDataType.STRUCT_ARRAY: if "sparse" in var.ncattrs(): shapes = self.group[var_name + ":shape"][()] - for index, node in tree_iter(self.ids, metadata): + for index, node in tree_iter(self.ids, target_metadata): node.resize(shapes[index][0]) else: @@ -132,7 +178,7 @@ def run(self) -> None: metadata.path_string, self.homogeneous_time )[-1] size = self.group.dimensions[dim].size - for _, node in tree_iter(self.ids, metadata): + for _, node in tree_iter(self.ids, target_metadata): node.resize(size) continue @@ -144,23 +190,30 @@ def run(self) -> None: if "sparse" in var.ncattrs(): if metadata.ndim: shapes = self.group[var_name + ":shape"][()] - for index, node in tree_iter(self.ids, metadata): + for index, node in tree_iter(self.ids, target_metadata): shape = shapes[index] if shape.all(): - node.value = data[index + tuple(map(slice, shapes[index]))] + # NOTE: bypassing IDSPrimitive.value.setter logic + node._IDSPrimitive__value = data[ + index + tuple(map(slice, shape)) + ] else: - for index, node in tree_iter(self.ids, metadata): + for index, node in tree_iter(self.ids, target_metadata): value = data[index] if value != getattr(var, "_FillValue", None): - node.value = data[index] + # NOTE: bypassing IDSPrimitive.value.setter logic + node._IDSPrimitive__value = value elif metadata.path_string not in self.ncmeta.aos: # Shortcut for assigning untensorized data - self.ids[metadata.path] = data + # Note: var[()] can return 0D numpy arrays. Instead of handling this + # here, we'll let IDSPrimitive.value.setter take care of it: + self.ids[target_metadata.path].value = data else: - for index, node in tree_iter(self.ids, metadata): - node.value = data[index] + for index, node in tree_iter(self.ids, target_metadata): + # NOTE: bypassing IDSPrimitive.value.setter logic + node._IDSPrimitive__value = data[index] def validate_variables(self) -> None: """Validate that all variables in the netCDF Group exist and match the DD.""" @@ -194,7 +247,7 @@ def validate_variables(self) -> None: # Check that the DD defines this variable, and validate its metadata var = self.group[var_name] try: - metadata = self.ids.metadata[var_name] + metadata = self.ids_metadata[var_name] except KeyError: raise InvalidNetCDFEntry( f"Invalid variable {var_name}: no such variable exists in the " @@ -300,3 +353,69 @@ def _validate_sparsity( raise variable_error( shape_var, "dtype", shape_var.dtype, "any integer type" ) + + +class LazyContext: + def __init__(self, nc2ids, index=()): + self.nc2ids = nc2ids + self.index = index + + def get_child(self, child): + metadata = child.metadata + path = metadata.path_string + data_type = metadata.data_type + nc2ids = self.nc2ids + var = nc2ids._lazy_map.get(path) + + if data_type is IDSDataType.STRUCT_ARRAY: + # Determine size of the aos + if var is None: + size = 0 + elif "sparse" in var.ncattrs(): + size = nc2ids.group[var.name + ":shape"][self.index][0] + else: + # FIXME: extract dimension name from nc file? + dim = nc2ids.ncmeta.get_dimensions( + metadata.path_string, nc2ids.homogeneous_time + )[-1] + size = nc2ids.group.dimensions[dim].size + + child._set_lazy_context(LazyArrayStructContext(nc2ids, self.index, size)) + + elif data_type is IDSDataType.STRUCTURE: + child._set_lazy_context(self) + + elif var is not None: # Data elements + value = None + if "sparse" in var.ncattrs(): + if metadata.ndim: + shape_var = nc2ids.group[var.name + ":shape"] + shape = shape_var[self.index] + if shape.all(): + value = var[self.index + tuple(map(slice, shape))] + else: + value = var[self.index] + if value == getattr(var, "_FillValue", None): + value = None # Skip setting + else: + value = var[self.index] + + if value is not None: + if isinstance(value, np.ndarray): + # Convert the numpy array to a read-only view + value = value.view() + value.flags.writeable = False + # NOTE: bypassing IDSPrimitive.value.setter logic + child._IDSPrimitive__value = value + + +class LazyArrayStructContext(LazyContext): + def __init__(self, nc2ids, index, size): + super().__init__(nc2ids, index) + self.size = size + + def get_context(self): + return self # IDSStructArray expects to get something with a size attribute + + def iterate_to_index(self, index: int) -> LazyContext: + return LazyContext(self.nc2ids, self.index + (index,)) diff --git a/imas/backends/netcdf/nc_validate.py b/imas/backends/netcdf/nc_validate.py index 2a4877d..03aded1 100644 --- a/imas/backends/netcdf/nc_validate.py +++ b/imas/backends/netcdf/nc_validate.py @@ -47,8 +47,9 @@ def validate_netcdf_file(filename: str) -> None: for ids_name in ids_names: for occurrence in entry.list_all_occurrences(ids_name): group = dataset[f"{ids_name}/{occurrence}"] + ids = factory.new(ids_name) try: - NC2IDS(group, factory.new(ids_name)).validate_variables() + NC2IDS(group, ids, ids.metadata, None).validate_variables() except InvalidNetCDFEntry as exc: occ = f":{occurrence}" if occurrence else "" raise InvalidNetCDFEntry(f"Invalid IDS {ids_name}{occ}: {exc}") diff --git a/imas/ids_structure.py b/imas/ids_structure.py index 3482d6e..2727003 100644 --- a/imas/ids_structure.py +++ b/imas/ids_structure.py @@ -6,11 +6,10 @@ import logging from copy import deepcopy from types import MappingProxyType -from typing import Generator, List, Optional, Union +from typing import TYPE_CHECKING, Generator, List, Optional, Union from xxhash import xxh3_64 -from imas.backends.imas_core.al_context import LazyALContext from imas.ids_base import IDSBase, IDSDoc from imas.ids_identifiers import IDSIdentifier from imas.ids_metadata import IDSDataType, IDSMetadata @@ -18,6 +17,9 @@ from imas.ids_primitive import IDSPrimitive from imas.ids_struct_array import IDSStructArray +if TYPE_CHECKING: + from imas.backends.imas_core.al_context import LazyALContext + logger = logging.getLogger(__name__) @@ -32,7 +34,7 @@ class IDSStructure(IDSBase): __doc__ = IDSDoc(__doc__) _children: "MappingProxyType[str, IDSMetadata]" - _lazy_context: Optional[LazyALContext] + _lazy_context: Optional["LazyALContext"] def __init__(self, parent: IDSBase, metadata: IDSMetadata): """Initialize IDSStructure from metadata specification @@ -62,10 +64,8 @@ def __getattr__(self, name): child_meta = self._children[name] child = child_meta._node_type(self, child_meta) self.__dict__[name] = child # bypass setattr logic below: avoid recursion - if self._lazy: # lazy load the child - from imas.backends.imas_core.db_entry_helpers import _get_child - - _get_child(child, self._lazy_context) + if self._lazy and self._lazy_context is not None: # lazy load the child + self._lazy_context.get_child(child) return child def _assign_identifier(self, value: Union[IDSIdentifier, str, int]) -> None: @@ -168,7 +168,7 @@ def __eq__(self, other) -> bool: return False # Not equal if there is any difference return True # Equal when there are no differences - def _set_lazy_context(self, ctx: LazyALContext) -> None: + def _set_lazy_context(self, ctx: "LazyALContext") -> None: """Called by DBEntry during a lazy get/get_slice. Set the context that we can use for retrieving our children. diff --git a/imas/test/test_lazy_loading.py b/imas/test/test_lazy_loading.py index fabc8a3..9023a79 100644 --- a/imas/test/test_lazy_loading.py +++ b/imas/test/test_lazy_loading.py @@ -3,7 +3,6 @@ import numpy import pytest - from imas.backends.imas_core.imas_interface import ll_interface from imas.db_entry import DBEntry from imas.ids_defs import ( @@ -22,6 +21,15 @@ def test_lazy_load_aos(backend, worker_id, tmp_path, log_lowlevel_calls): if backend == ASCII_BACKEND: pytest.skip("Lazy loading is not supported by the ASCII backend.") dbentry = open_dbentry(backend, "w", worker_id, tmp_path, dd_version="3.39.0") + run_lazy_load_aos(dbentry) + + +def test_lazy_load_aos_netcdf(tmp_path): + dbentry = DBEntry(str(tmp_path / "lazy_load_aos.nc"), "x", dd_version="3.39.0") + run_lazy_load_aos(dbentry) + + +def run_lazy_load_aos(dbentry): ids = dbentry.factory.new("core_profiles") ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS ids.profiles_1d.resize(10) @@ -46,9 +54,12 @@ def test_lazy_load_aos(backend, worker_id, tmp_path, log_lowlevel_calls): assert values[method].call_count == 0 # Test get_slice - lazy_ids_slice = dbentry.get_slice("core_profiles", 3.5, PREVIOUS_INTERP, lazy=True) - assert lazy_ids_slice.profiles_1d.shape == (1,) - assert lazy_ids_slice.profiles_1d[0].time == 3 + try: + lazy_slice = dbentry.get_slice("core_profiles", 3.5, PREVIOUS_INTERP, lazy=True) + assert lazy_slice.profiles_1d.shape == (1,) + assert lazy_slice.profiles_1d[0].time == 3 + except NotImplementedError: + pass # netCDF backend doesn't implement get_slice dbentry.close() @@ -57,6 +68,15 @@ def test_lazy_loading_distributions_random(backend, worker_id, tmp_path): if backend == ASCII_BACKEND: pytest.skip("Lazy loading is not supported by the ASCII backend.") dbentry = open_dbentry(backend, "w", worker_id, tmp_path) + run_lazy_loading_distributions_random(dbentry) + + +def test_lazy_loading_distributions_random_netcdf(tmp_path): + dbentry = DBEntry(str(tmp_path / "lazy_load_distributions.nc"), "x") + run_lazy_loading_distributions_random(dbentry) + + +def run_lazy_loading_distributions_random(dbentry): ids = IDSFactory().new("distributions") fill_consistent(ids) dbentry.put(ids) @@ -92,7 +112,15 @@ def test_lazy_load_close_dbentry(requires_imas): def test_lazy_load_readonly(requires_imas): dbentry = DBEntry(MEMORY_BACKEND, "ITER", 1, 1) dbentry.create() + run_lazy_load_readonly(dbentry) + + +def test_lazy_load_readonly_netcdf(tmp_path): + dbentry = DBEntry(str(tmp_path / "lazy_load_readonly.nc"), "x") + run_lazy_load_readonly(dbentry) + +def run_lazy_load_readonly(dbentry): ids = dbentry.factory.core_profiles() ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS ids.time = [1, 2] @@ -165,6 +193,27 @@ def test_lazy_load_with_new_aos(requires_imas): dbentry.close() +def test_lazy_load_with_new_aos_netcdf(tmp_path): + fname = str(tmp_path / "new_aos.nc") + with DBEntry(fname, "x", dd_version="3.30.0") as dbentry: + et = dbentry.factory.edge_transport() + + et.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + et.time = [1.0] + et.model.resize(1) + et.model[0].ggd.resize(1) + et.model[0].ggd[0].electrons.particles.d.resize(1) + et.model[0].ggd[0].electrons.particles.d[0].grid_index = -1 + dbentry.put(et) + + with DBEntry(fname, "r", dd_version="3.39.0") as entry2: + lazy_et = entry2.get("edge_transport", lazy=True) + assert numpy.array_equal(lazy_et.time, [1.0]) + assert lazy_et.model[0].ggd[0].electrons.particles.d[0].grid_index == -1 + # d_radial did not exist in 3.30.0 + assert len(lazy_et.model[0].ggd[0].electrons.particles.d_radial) == 0 + + def test_lazy_load_with_new_structure(requires_imas): dbentry = DBEntry(MEMORY_BACKEND, "ITER", 1, 1, dd_version="3.30.0") dbentry.create() diff --git a/imas/test/test_nbc_change.py b/imas/test/test_nbc_change.py index b5c7905..91ede0e 100644 --- a/imas/test/test_nbc_change.py +++ b/imas/test/test_nbc_change.py @@ -9,16 +9,11 @@ import numpy as np import pytest - from imas.db_entry import DBEntry from imas.ids_convert import convert_ids from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS, MEMORY_BACKEND from imas.ids_factory import IDSFactory -from imas.test.test_helpers import ( - compare_children, - fill_with_random_data, - open_dbentry, -) +from imas.test.test_helpers import compare_children, fill_with_random_data, open_dbentry @pytest.fixture(autouse=True) @@ -97,6 +92,23 @@ def test_nbc_0d_to_1d(caplog, requires_imas): entry_339.close() +def test_nbc_0d_to_1d_netcdf(caplog, tmp_path): + # channel/filter_spectrometer/radiance_calibration in spectrometer visible changed + # from FLT_0D to FLT_1D in DD 3.39.0 + ids = IDSFactory("3.32.0").spectrometer_visible() + ids.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + ids.channel.resize(1) + ids.channel[0].filter_spectrometer.radiance_calibration = 1.0 + + # Test implicit conversion during get + with DBEntry(str(tmp_path / "test.nc"), "x", dd_version="3.32.0") as entry_332: + entry_332.put(ids) + with DBEntry(str(tmp_path / "test.nc"), "r", dd_version="3.39.0") as entry_339: + ids_339 = entry_339.get("spectrometer_visible") # implicit conversion + assert not ids_339.channel[0].filter_spectrometer.radiance_calibration.has_value + entry_339.close() + + def test_nbc_change_aos_renamed(): """Test renamed AoS in pulse_schedule: ec/antenna -> ec/launcher. @@ -272,7 +284,7 @@ def test_pulse_schedule_aos_renamed_autofill_up(backend, worker_id, tmp_path): dbentry.close() -def test_pulse_schedule_multi_rename(): +def test_pulse_schedule_multi_rename(tmp_path): # Multiple renames of the same element: # DD >= 3.40+: ec/beam # DD 3.26-3.40: ec/launcher (but NBC metadata added in 3.28 only) @@ -294,9 +306,18 @@ def test_pulse_schedule_multi_rename(): ps["3.40.0"].ec.beam[0].name = name for version1 in ps: + ncfilename = str(tmp_path / f"{version1}.nc") + with DBEntry(ncfilename, "x", dd_version=version1) as entry: + entry.put(ps[version1]) + for version2 in ps: converted = convert_ids(ps[version1], version2) - compare_children(ps[version2], converted) + compare_children(ps[version2].ec, converted.ec) + + # Test with netCDF backend + with DBEntry(ncfilename, "r", dd_version=version2) as entry: + converted = entry.get("pulse_schedule") + compare_children(ps[version2].ec, converted.ec) def test_autofill_save_newer(ids_name, backend, worker_id, tmp_path): diff --git a/imas/test/test_nc_validation.py b/imas/test/test_nc_validation.py index 2f63b01..69f9f01 100644 --- a/imas/test/test_nc_validation.py +++ b/imas/test/test_nc_validation.py @@ -1,7 +1,6 @@ import netCDF4 import numpy as np import pytest - from imas.backends.netcdf.ids2nc import IDS2NC from imas.backends.netcdf.nc2ids import NC2IDS from imas.backends.netcdf.nc_validate import validate_netcdf_file @@ -32,7 +31,8 @@ def memfile_with_ids(memfile, factory): ids.profiles_1d[0].zeff = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] IDS2NC(ids, memfile).run() # This one is valid: - NC2IDS(memfile, factory.core_profiles()).run() + ids = factory.core_profiles() + NC2IDS(memfile, ids, ids.metadata, None).run(lazy=False) return memfile @@ -51,66 +51,75 @@ def test_invalid_homogeneous_time(memfile, factory): ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(empty_group, ids) # ids_properties.homogeneous_time does not exist + # ids_properties.homogeneous_time does not exist + NC2IDS(empty_group, ids, ids.metadata, None) with pytest.raises(InvalidNetCDFEntry): - NC2IDS(invalid_dtype, ids) + NC2IDS(invalid_dtype, ids, ids.metadata, None) with pytest.raises(InvalidNetCDFEntry): - NC2IDS(invalid_shape, ids) + NC2IDS(invalid_shape, ids, ids.metadata, None) with pytest.raises(InvalidNetCDFEntry): - NC2IDS(invalid_value, ids) + NC2IDS(invalid_value, ids, ids.metadata, None) def test_invalid_units(memfile_with_ids, factory): memfile_with_ids["time"].units = "hours" + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_invalid_documentation(memfile_with_ids, factory, caplog): + ids = factory.core_profiles() with caplog.at_level("WARNING"): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) assert not caplog.records # Invalid docstring logs a warning memfile_with_ids["time"].documentation = "https://en.wikipedia.org/wiki/Time" with caplog.at_level("WARNING"): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) assert len(caplog.records) == 1 def test_invalid_dimension_name(memfile_with_ids, factory): memfile_with_ids.renameDimension("time", "T") + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_invalid_coordinates(memfile_with_ids, factory): memfile_with_ids["profiles_1d.grid.rho_tor_norm"].coordinates = "xyz" + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_invalid_ancillary_variables(memfile_with_ids, factory): memfile_with_ids["time"].ancillary_variables = "xyz" + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_extra_attributes(memfile_with_ids, factory): memfile_with_ids["time"].new_attribute = [1, 2, 3] + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_shape_array_without_data(memfile_with_ids, factory): memfile_with_ids.createVariable("profiles_1d.t_i_average:shape", int, ()) + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_shape_array_without_sparse_data(memfile_with_ids, factory): memfile_with_ids.createVariable("profiles_1d.grid.rho_tor_norm:shape", int, ()) + ids = factory.core_profiles() with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, factory.core_profiles()).run() + NC2IDS(memfile_with_ids, ids, ids.metadata, None).run(lazy=False) def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory): @@ -128,7 +137,7 @@ def test_shape_array_with_invalid_dimensions(memfile_with_ids, factory): ("time", "profiles_1d.grid.rho_tor_norm:i"), ) with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, cp).run() + NC2IDS(memfile_with_ids, cp, cp.metadata, None).run(lazy=False) def test_shape_array_with_invalid_dtype(memfile_with_ids, factory): @@ -144,7 +153,7 @@ def test_shape_array_with_invalid_dtype(memfile_with_ids, factory): "profiles_1d.t_i_average:shape", float, ("time", "1D") ) with pytest.raises(InvalidNetCDFEntry): - NC2IDS(memfile_with_ids, cp).run() + NC2IDS(memfile_with_ids, cp, cp.metadata, None).run(lazy=False) def test_validate_nc(tmpdir):