From 18138a807e14a6e4604c608abeeff61c88f9d5d7 Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Mon, 20 Jan 2025 23:34:52 +0000 Subject: [PATCH 01/11] Modifications for compatibility with TORAX. --- imaspy/backends/netcdf/db_entry_nc.py | 7 +++++++ imaspy/backends/netcdf/ids2nc.py | 6 +++++- imaspy/backends/netcdf/nc_validate.py | 9 +++++---- imaspy/ids_primitive.py | 2 +- imaspy/test/test_cli.py | 6 ++++-- imaspy/test/test_dbentry.py | 1 + imaspy/test/test_ids_mixin.py | 3 +++ imaspy/test/test_ids_toplevel.py | 2 +- imaspy/test/test_minimal_types.py | 5 +++-- imaspy/test/test_nbc_change.py | 2 +- imaspy/test/test_static_ids.py | 2 +- imaspy/test/test_util.py | 6 +++--- 12 files changed, 35 insertions(+), 16 deletions(-) diff --git a/imaspy/backends/netcdf/db_entry_nc.py b/imaspy/backends/netcdf/db_entry_nc.py index 732eb97d..c008262c 100644 --- a/imaspy/backends/netcdf/db_entry_nc.py +++ b/imaspy/backends/netcdf/db_entry_nc.py @@ -33,12 +33,19 @@ def __init__(self, fname: str, mode: str, factory: IDSFactory) -> None: "The `netCDF4` python module is not available. Please install this " "module to read/write IMAS netCDF files with IMASPy." ) + # To support netcdf v1.4 (which has no mode "x") we map it to "w" with `clobber=True`. + if mode == "x": + mode = "w" + clobber = False + else: + clobber = True self._dataset = netCDF4.Dataset( fname, mode, format="NETCDF4", auto_complex=True, + clobber=clobber, ) """NetCDF4 dataset.""" self._factory = factory diff --git a/imaspy/backends/netcdf/ids2nc.py b/imaspy/backends/netcdf/ids2nc.py index 34e63101..61e42cf2 100644 --- a/imaspy/backends/netcdf/ids2nc.py +++ b/imaspy/backends/netcdf/ids2nc.py @@ -7,6 +7,7 @@ import netCDF4 import numpy +from packaging import version from imaspy.backends.netcdf.nc_metadata import NCMetadata from imaspy.ids_base import IDSBase @@ -187,7 +188,10 @@ def create_variables(self) -> None: dtype = dtypes[metadata.data_type] kwargs = {} if dtype is not str: # Enable compression: - kwargs.update(compression="zlib", complevel=1) + if version.parse(netCDF4.__version__) > version.parse("1.4.1"): + kwargs.update(compression="zlib", complevel=1) + else: + kwargs.update(zlib=True, complevel=1) if dtype is not dtypes[IDSDataType.CPX]: # Set fillvalue kwargs.update(fill_value=default_fillvals[metadata.data_type]) # Create variable diff --git a/imaspy/backends/netcdf/nc_validate.py b/imaspy/backends/netcdf/nc_validate.py index 49a14283..07a7ad78 100644 --- a/imaspy/backends/netcdf/nc_validate.py +++ b/imaspy/backends/netcdf/nc_validate.py @@ -23,23 +23,24 @@ def validate_netcdf_file(filename: str) -> None: # additional variables are smuggled inside: groups = [dataset] + [dataset[group] for group in dataset.groups] for group in groups: + group_name = group.path.split('/')[-1] if group.variables or group.dimensions: raise InvalidNetCDFEntry( "NetCDF file should not have variables or dimensions in the " - f"{group.name} group." + f"{group_name} group." ) if group is dataset: continue - if group.name not in ids_names: + if group_name not in ids_names: raise InvalidNetCDFEntry( - f"Invalid group name {group.name}: there is no IDS with this name." + f"Invalid group name {group_name}: there is no IDS with this name." ) for subgroup in group.groups: try: int(subgroup) except ValueError: raise InvalidNetCDFEntry( - f"Invalid group name {group.name}/{subgroup}: " + f"Invalid group name {group_name}/{subgroup}: " f"{subgroup} is not a valid occurrence number." ) diff --git a/imaspy/ids_primitive.py b/imaspy/ids_primitive.py index 94f865b6..e27eb93f 100644 --- a/imaspy/ids_primitive.py +++ b/imaspy/ids_primitive.py @@ -481,7 +481,7 @@ def _cast_value(self, value): value = np.asanyarray(value) if value.dtype != dtype: logger.info(_CONVERT_MSG, value.dtype, self) - value = np.array(value, dtype=dtype, copy=False) + value = np.asarray(value, dtype=dtype,) if value.ndim != self.metadata.ndim: raise ValueError(f"Trying to assign a {value.ndim}D value to {self!r}.") return value diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index 604a7f7e..f9ee5383 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -4,6 +4,7 @@ from click.testing import CliRunner from packaging.version import Version +from imaspy.backends.imas_core.imas_interface import has_imas from imaspy.backends.imas_core.imas_interface import ll_interface from imaspy.command.cli import print_version from imaspy.command.db_analysis import analyze_db, process_db_analysis @@ -12,6 +13,7 @@ @pytest.mark.cli +@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") def test_imaspy_version(): runner = CliRunner() result = runner.invoke(print_version) @@ -19,8 +21,8 @@ def test_imaspy_version(): @pytest.mark.cli -@pytest.mark.skipif(ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5") -def test_db_analysis(tmp_path): +@pytest.mark.skipif(not has_imas or ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5 AND Requires IMAS Core.") +def test_db_analysis(tmp_path,): # This only tests the happy flow, error handling is not tested db_path = tmp_path / "test_db_analysis" with DBEntry(f"imas:hdf5?path={db_path}", "w") as entry: diff --git a/imaspy/test/test_dbentry.py b/imaspy/test/test_dbentry.py index 2d82af36..d67fae0d 100644 --- a/imaspy/test/test_dbentry.py +++ b/imaspy/test/test_dbentry.py @@ -82,6 +82,7 @@ def test_dbentry_constructor(): assert get_entry_attrs(entry) == (1, 2, 3, 4, None, 6) +@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path): entry = open_dbentry(imaspy.ids_defs.MEMORY_BACKEND, "w", worker_id, tmp_path) ids = entry.factory.core_profiles() diff --git a/imaspy/test/test_ids_mixin.py b/imaspy/test/test_ids_mixin.py index 164adcdd..2b3f7b03 100644 --- a/imaspy/test/test_ids_mixin.py +++ b/imaspy/test/test_ids_mixin.py @@ -1,7 +1,10 @@ # This file is part of IMASPy. # You should have received the IMASPy LICENSE file with this project. +import pytest +from imaspy.backends.imas_core.imas_interface import has_imas +@pytest.mark.skipif(has_imas, reason="Requires IMAS Core.") def test_toplevel(fake_filled_toplevel): top = fake_filled_toplevel assert top.wavevector._toplevel == top diff --git a/imaspy/test/test_ids_toplevel.py b/imaspy/test/test_ids_toplevel.py index 4721f3c3..0e8d8c32 100644 --- a/imaspy/test/test_ids_toplevel.py +++ b/imaspy/test/test_ids_toplevel.py @@ -46,7 +46,7 @@ def test_pretty_print(ids): assert pprint.pformat(ids) == "" -def test_serialize_nondefault_dd_version(): +def test_serialize_nondefault_dd_version(requires_imas): ids = IDSFactory("3.31.0").core_profiles() fill_with_random_data(ids) data = ids.serialize() diff --git a/imaspy/test/test_minimal_types.py b/imaspy/test/test_minimal_types.py index ee38761c..0bb9ac30 100644 --- a/imaspy/test/test_minimal_types.py +++ b/imaspy/test/test_minimal_types.py @@ -1,5 +1,6 @@ # A minimal testcase loading an IDS file and checking that the structure built is ok from numbers import Complex, Integral, Number, Real +from packaging import version import numpy as np import pytest @@ -61,7 +62,7 @@ def test_assign_str_1d(minimal, caplog): # Prevent the expected numpy ComplexWarnings from cluttering pytest output -@pytest.mark.filterwarnings("ignore::numpy.ComplexWarning") +@pytest.mark.filterwarnings("ignore::numpy.ComplexWarning" if version.parse(np.__version__) < version.parse("2.0.0") else "ignore::numpy.exceptions.ComplexWarning") @pytest.mark.parametrize("typ, max_dim", [("flt", 6), ("cpx", 6), ("int", 3)]) def test_assign_numeric_types(minimal, caplog, typ, max_dim): caplog.set_level("INFO", "imaspy") @@ -87,7 +88,7 @@ def test_assign_numeric_types(minimal, caplog, typ, max_dim): len(caplog.records) == 1 elif dim == other_ndim >= 1 and other_typ == "cpx": # np allows casting of complex to float or int, but warns: - with pytest.warns(np.ComplexWarning): + with pytest.warns(np.ComplexWarning if version.parse(np.__version__) < version.parse("2.0.0") else np.exceptions.ComplexWarning): caplog.clear() minimal[name].value = value assert len(caplog.records) == 1 diff --git a/imaspy/test/test_nbc_change.py b/imaspy/test/test_nbc_change.py index cbcf3f58..2e328982 100644 --- a/imaspy/test/test_nbc_change.py +++ b/imaspy/test/test_nbc_change.py @@ -54,7 +54,7 @@ def test_nbc_structure_to_aos(caplog): assert caplog.record_tuples[0][:2] == ("imaspy.ids_convert", logging.WARNING) -def test_nbc_0d_to_1d(caplog): +def test_nbc_0d_to_1d(caplog, requires_imas): # channel/filter_spectrometer/radiance_calibration in spectrometer visible changed # from FLT_0D to FLT_1D in DD 3.39.0 ids = IDSFactory("3.32.0").spectrometer_visible() diff --git a/imaspy/test/test_static_ids.py b/imaspy/test/test_static_ids.py index 1f430c10..680ecd2b 100644 --- a/imaspy/test/test_static_ids.py +++ b/imaspy/test/test_static_ids.py @@ -21,7 +21,7 @@ def test_ids_valid_type(): assert ids_types in ({IDSType.NONE}, {IDSType.CONSTANT, IDSType.DYNAMIC}) -def test_constant_ids(caplog): +def test_constant_ids(caplog, requires_imas): ids = imaspy.IDSFactory().new("amns_data") if ids.metadata.type is IDSType.NONE: pytest.skip("IDS definition has no constant IDSs") diff --git a/imaspy/test/test_util.py b/imaspy/test/test_util.py index 37c419a0..2c4dad97 100644 --- a/imaspy/test/test_util.py +++ b/imaspy/test/test_util.py @@ -54,7 +54,7 @@ def test_inspect(): inspect(cp.profiles_1d[1].grid.rho_tor_norm) # IDSPrimitive -def test_inspect_lazy(): +def test_inspect_lazy(requires_imas): with get_training_db_entry() as entry: cp = entry.get("core_profiles", lazy=True) inspect(cp) @@ -141,7 +141,7 @@ def test_idsdiffgen(): assert diff[0] == ("profiles_1d/time", -1, 0) -def test_idsdiff(): +def test_idsdiff(requires_imas): # Test the diff rendering for two sample IDSs with get_training_db_entry() as entry: imaspy.util.idsdiff(entry.get("core_profiles"), entry.get("equilibrium")) @@ -179,7 +179,7 @@ def test_get_toplevel(): assert get_toplevel(cp) is cp -def test_is_lazy_loaded(): +def test_is_lazy_loaded(requires_imas): with get_training_db_entry() as entry: assert is_lazy_loaded(entry.get("core_profiles")) is False assert is_lazy_loaded(entry.get("core_profiles", lazy=True)) is True From 046891a8ab6f6d2a9a0648b0d9d4e86ad29b2201 Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Mon, 20 Jan 2025 23:54:44 +0000 Subject: [PATCH 02/11] Modify tests to use fixture. --- imaspy/test/test_cli.py | 3 +-- imaspy/test/test_dbentry.py | 3 +-- imaspy/test/test_ids_mixin.py | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index f9ee5383..db7c462f 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -13,8 +13,7 @@ @pytest.mark.cli -@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") -def test_imaspy_version(): +def test_imaspy_version(requires_imas): runner = CliRunner() result = runner.invoke(print_version) assert result.exit_code == 0 diff --git a/imaspy/test/test_dbentry.py b/imaspy/test/test_dbentry.py index d67fae0d..cb7ebe12 100644 --- a/imaspy/test/test_dbentry.py +++ b/imaspy/test/test_dbentry.py @@ -82,8 +82,7 @@ def test_dbentry_constructor(): assert get_entry_attrs(entry) == (1, 2, 3, 4, None, 6) -@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") -def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path): +def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path, requires_imas): entry = open_dbentry(imaspy.ids_defs.MEMORY_BACKEND, "w", worker_id, tmp_path) ids = entry.factory.core_profiles() ids.ids_properties.homogeneous_time = 0 diff --git a/imaspy/test/test_ids_mixin.py b/imaspy/test/test_ids_mixin.py index 2b3f7b03..675e2575 100644 --- a/imaspy/test/test_ids_mixin.py +++ b/imaspy/test/test_ids_mixin.py @@ -1,11 +1,8 @@ # This file is part of IMASPy. # You should have received the IMASPy LICENSE file with this project. -import pytest -from imaspy.backends.imas_core.imas_interface import has_imas -@pytest.mark.skipif(has_imas, reason="Requires IMAS Core.") -def test_toplevel(fake_filled_toplevel): +def test_toplevel(fake_filled_toplevel, requires_imas): top = fake_filled_toplevel assert top.wavevector._toplevel == top assert top.wavevector[0].radial_component_norm._toplevel == top From 82578736e3c86574f99103a165e32f12ed7182d3 Mon Sep 17 00:00:00 2001 From: Anushan Fernando <35841118+Nush395@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:27:25 +0000 Subject: [PATCH 03/11] Update imaspy/test/test_ids_mixin.py Co-authored-by: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> --- imaspy/test/test_ids_mixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imaspy/test/test_ids_mixin.py b/imaspy/test/test_ids_mixin.py index 675e2575..164adcdd 100644 --- a/imaspy/test/test_ids_mixin.py +++ b/imaspy/test/test_ids_mixin.py @@ -2,7 +2,7 @@ # You should have received the IMASPy LICENSE file with this project. -def test_toplevel(fake_filled_toplevel, requires_imas): +def test_toplevel(fake_filled_toplevel): top = fake_filled_toplevel assert top.wavevector._toplevel == top assert top.wavevector[0].radial_component_norm._toplevel == top From bd6a87eb0a4a28b08a4f37603fb4b240dc8f4eaa Mon Sep 17 00:00:00 2001 From: Anushan Fernando <35841118+Nush395@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:27:45 +0000 Subject: [PATCH 04/11] Update imaspy/test/test_cli.py Co-authored-by: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> --- imaspy/test/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index db7c462f..e6a420c7 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -21,7 +21,7 @@ def test_imaspy_version(requires_imas): @pytest.mark.cli @pytest.mark.skipif(not has_imas or ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5 AND Requires IMAS Core.") -def test_db_analysis(tmp_path,): +def test_db_analysis(tmp_path): # This only tests the happy flow, error handling is not tested db_path = tmp_path / "test_db_analysis" with DBEntry(f"imas:hdf5?path={db_path}", "w") as entry: From 365afdcfb1f6450efa69e4a4f2bc69649b0c100a Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 10:52:21 +0000 Subject: [PATCH 05/11] Add error message when attemtping to store complex number with netcdf<1.7.0. --- imaspy/backends/netcdf/ids2nc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imaspy/backends/netcdf/ids2nc.py b/imaspy/backends/netcdf/ids2nc.py index 61e42cf2..45a04d6b 100644 --- a/imaspy/backends/netcdf/ids2nc.py +++ b/imaspy/backends/netcdf/ids2nc.py @@ -10,6 +10,7 @@ from packaging import version from imaspy.backends.netcdf.nc_metadata import NCMetadata +from imaspy.exception import InvalidNetCDFEntry from imaspy.ids_base import IDSBase from imaspy.ids_data_type import IDSDataType from imaspy.ids_defs import IDS_TIME_MODE_HOMOGENEOUS @@ -186,6 +187,8 @@ def create_variables(self) -> None: else: dtype = dtypes[metadata.data_type] + if version.parse(netCDF4.__version__) < version.parse("1.7.0") and dtype is dtypes[IDSDataType.CPX]: + raise InvalidNetCDFEntry(f"Found complex data in {var_name}, NetCDF 1.7.0 or later is required for complex data types") kwargs = {} if dtype is not str: # Enable compression: if version.parse(netCDF4.__version__) > version.parse("1.4.1"): From fd6ac2ea7b859727a3d1e3e3b2ffa6f7b2086d78 Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 10:57:10 +0000 Subject: [PATCH 06/11] Formatting. --- imaspy/backends/netcdf/db_entry_nc.py | 3 ++- imaspy/backends/netcdf/ids2nc.py | 10 ++++++++-- imaspy/backends/netcdf/nc_validate.py | 2 +- imaspy/ids_primitive.py | 5 ++++- imaspy/test/test_cli.py | 5 ++++- imaspy/test/test_minimal_types.py | 12 ++++++++++-- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/imaspy/backends/netcdf/db_entry_nc.py b/imaspy/backends/netcdf/db_entry_nc.py index c008262c..97d5dffe 100644 --- a/imaspy/backends/netcdf/db_entry_nc.py +++ b/imaspy/backends/netcdf/db_entry_nc.py @@ -33,7 +33,8 @@ def __init__(self, fname: str, mode: str, factory: IDSFactory) -> None: "The `netCDF4` python module is not available. Please install this " "module to read/write IMAS netCDF files with IMASPy." ) - # To support netcdf v1.4 (which has no mode "x") we map it to "w" with `clobber=True`. + # To support netcdf v1.4 (which has no mode "x") we map it to "w" with + # `clobber=True`. if mode == "x": mode = "w" clobber = False diff --git a/imaspy/backends/netcdf/ids2nc.py b/imaspy/backends/netcdf/ids2nc.py index 45a04d6b..0328b635 100644 --- a/imaspy/backends/netcdf/ids2nc.py +++ b/imaspy/backends/netcdf/ids2nc.py @@ -187,8 +187,14 @@ def create_variables(self) -> None: else: dtype = dtypes[metadata.data_type] - if version.parse(netCDF4.__version__) < version.parse("1.7.0") and dtype is dtypes[IDSDataType.CPX]: - raise InvalidNetCDFEntry(f"Found complex data in {var_name}, NetCDF 1.7.0 or later is required for complex data types") + if ( + version.parse(netCDF4.__version__) < version.parse("1.7.0") + and dtype is dtypes[IDSDataType.CPX] + ): + raise InvalidNetCDFEntry( + f"Found complex data in {var_name}, NetCDF 1.7.0 or" + f" later is required for complex data types" + ) kwargs = {} if dtype is not str: # Enable compression: if version.parse(netCDF4.__version__) > version.parse("1.4.1"): diff --git a/imaspy/backends/netcdf/nc_validate.py b/imaspy/backends/netcdf/nc_validate.py index 07a7ad78..f7528a8a 100644 --- a/imaspy/backends/netcdf/nc_validate.py +++ b/imaspy/backends/netcdf/nc_validate.py @@ -23,7 +23,7 @@ def validate_netcdf_file(filename: str) -> None: # additional variables are smuggled inside: groups = [dataset] + [dataset[group] for group in dataset.groups] for group in groups: - group_name = group.path.split('/')[-1] + group_name = group.path.split("/")[-1] if group.variables or group.dimensions: raise InvalidNetCDFEntry( "NetCDF file should not have variables or dimensions in the " diff --git a/imaspy/ids_primitive.py b/imaspy/ids_primitive.py index e27eb93f..71b1744a 100644 --- a/imaspy/ids_primitive.py +++ b/imaspy/ids_primitive.py @@ -481,7 +481,10 @@ def _cast_value(self, value): value = np.asanyarray(value) if value.dtype != dtype: logger.info(_CONVERT_MSG, value.dtype, self) - value = np.asarray(value, dtype=dtype,) + value = np.asarray( + value, + dtype=dtype, + ) if value.ndim != self.metadata.ndim: raise ValueError(f"Trying to assign a {value.ndim}D value to {self!r}.") return value diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index e6a420c7..fdea00f4 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -20,7 +20,10 @@ def test_imaspy_version(requires_imas): @pytest.mark.cli -@pytest.mark.skipif(not has_imas or ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5 AND Requires IMAS Core.") +@pytest.mark.skipif( + not has_imas or ll_interface._al_version < Version("5.0"), + reason="Needs AL >= 5 AND Requires IMAS Core.", +) def test_db_analysis(tmp_path): # This only tests the happy flow, error handling is not tested db_path = tmp_path / "test_db_analysis" diff --git a/imaspy/test/test_minimal_types.py b/imaspy/test/test_minimal_types.py index 0bb9ac30..d4614de5 100644 --- a/imaspy/test/test_minimal_types.py +++ b/imaspy/test/test_minimal_types.py @@ -62,7 +62,11 @@ def test_assign_str_1d(minimal, caplog): # Prevent the expected numpy ComplexWarnings from cluttering pytest output -@pytest.mark.filterwarnings("ignore::numpy.ComplexWarning" if version.parse(np.__version__) < version.parse("2.0.0") else "ignore::numpy.exceptions.ComplexWarning") +@pytest.mark.filterwarnings( + "ignore::numpy.ComplexWarning" + if version.parse(np.__version__) < version.parse("2.0.0") + else "ignore::numpy.exceptions.ComplexWarning" +) @pytest.mark.parametrize("typ, max_dim", [("flt", 6), ("cpx", 6), ("int", 3)]) def test_assign_numeric_types(minimal, caplog, typ, max_dim): caplog.set_level("INFO", "imaspy") @@ -88,7 +92,11 @@ def test_assign_numeric_types(minimal, caplog, typ, max_dim): len(caplog.records) == 1 elif dim == other_ndim >= 1 and other_typ == "cpx": # np allows casting of complex to float or int, but warns: - with pytest.warns(np.ComplexWarning if version.parse(np.__version__) < version.parse("2.0.0") else np.exceptions.ComplexWarning): + with pytest.warns( + np.ComplexWarning + if version.parse(np.__version__) < version.parse("2.0.0") + else np.exceptions.ComplexWarning + ): caplog.clear() minimal[name].value = value assert len(caplog.records) == 1 From 8c45a9ebb4325fecbdcb60f1dd44c6dba012e82e Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 13:56:46 +0000 Subject: [PATCH 07/11] Add tests for different versions of netcdf. --- conftest.py | 14 ++++++++++ imaspy/test/test_cli.py | 7 +++-- imaspy/test/test_dbentry.py | 3 ++- imaspy/test/test_helpers.py | 34 ++++++++++++++++++------- imaspy/test/test_nc_autofill.py | 45 +++++++++++++++++++++++++++++++-- pyproject.toml | 2 +- 6 files changed, 90 insertions(+), 15 deletions(-) diff --git a/conftest.py b/conftest.py index 20b26679..d1893f76 100644 --- a/conftest.py +++ b/conftest.py @@ -7,6 +7,7 @@ # - Fixtures that are useful across test modules import functools +import importlib import logging import os import sys @@ -72,6 +73,19 @@ def pytest_addoption(parser): } +# This is a dummy fixture, usually provided by pytest-xdist that isn't available +# in google3. +# The `worker_id` is only used by tests that require IMAS Core which we never +# run +try: + import pytest_xdist +except ImportError: + # If pytest-xdist is not available we provide a dummy worker_id fixture. + @pytest.fixture() + def worker_id(): + return "master" + + @pytest.fixture(params=_BACKENDS) def backend(pytestconfig: pytest.Config, request: pytest.FixtureRequest): backends_provided = any(map(pytestconfig.getoption, _BACKENDS)) diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index fdea00f4..810acda6 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -13,7 +13,8 @@ @pytest.mark.cli -def test_imaspy_version(requires_imas): +@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") +def test_imaspy_version(): runner = CliRunner() result = runner.invoke(print_version) assert result.exit_code == 0 @@ -24,7 +25,9 @@ def test_imaspy_version(requires_imas): not has_imas or ll_interface._al_version < Version("5.0"), reason="Needs AL >= 5 AND Requires IMAS Core.", ) -def test_db_analysis(tmp_path): +def test_db_analysis( + tmp_path, +): # This only tests the happy flow, error handling is not tested db_path = tmp_path / "test_db_analysis" with DBEntry(f"imas:hdf5?path={db_path}", "w") as entry: diff --git a/imaspy/test/test_dbentry.py b/imaspy/test/test_dbentry.py index cb7ebe12..d67fae0d 100644 --- a/imaspy/test/test_dbentry.py +++ b/imaspy/test/test_dbentry.py @@ -82,7 +82,8 @@ def test_dbentry_constructor(): assert get_entry_attrs(entry) == (1, 2, 3, 4, None, 6) -def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path, requires_imas): +@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") +def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path): entry = open_dbentry(imaspy.ids_defs.MEMORY_BACKEND, "w", worker_id, tmp_path) ids = entry.factory.core_profiles() ids.ids_properties.homogeneous_time = 0 diff --git a/imaspy/test/test_helpers.py b/imaspy/test/test_helpers.py index 63a1cf79..8a651d93 100644 --- a/imaspy/test/test_helpers.py +++ b/imaspy/test/test_helpers.py @@ -93,7 +93,9 @@ def fill_with_random_data(structure, max_children=3): child.value = random_data(child.metadata.data_type, child.metadata.ndim) -def maybe_set_random_value(primitive: IDSPrimitive, leave_empty: float) -> None: +def maybe_set_random_value( + primitive: IDSPrimitive, leave_empty: float, skip_complex: bool +) -> None: """Set the value of an IDS primitive with a certain chance. If the IDSPrimitive has coordinates, then the size of the coordinates is taken into @@ -153,7 +155,7 @@ def maybe_set_random_value(primitive: IDSPrimitive, leave_empty: float) -> None: # Scale chance of not setting a coordinate by our number of dimensions, # such that overall there is roughly a 50% chance that any coordinate # remains empty - maybe_set_random_value(coordinate_element, 0.5**ndim) + maybe_set_random_value(coordinate_element, 0.5**ndim, skip_complex) size = coordinate_element.shape[0 if coordinate.references else dim] if coordinate.size: # coordinateX = OR 1...1 @@ -176,13 +178,18 @@ def maybe_set_random_value(primitive: IDSPrimitive, leave_empty: float) -> None: elif primitive.metadata.data_type is IDSDataType.FLT: primitive.value = np.random.random_sample(size=shape) elif primitive.metadata.data_type is IDSDataType.CPX: + if skip_complex: + # If we are skipping complex numbers then leave the value empty. + return val = np.random.random_sample(shape) + 1j * np.random.random_sample(shape) primitive.value = val else: raise ValueError(f"Invalid IDS data type: {primitive.metadata.data_type}") -def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): +def fill_consistent( + structure: IDSStructure, leave_empty: float = 0.2, skip_complex: bool = False +): """Fill a structure with random data, such that coordinate sizes are consistent. Sets homogeneous_time to heterogeneous (always). @@ -196,6 +203,9 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): exclusive_coordinates: list of IDSPrimitives that have exclusive alternative coordinates. These are initially not filled, and only at the very end of filling an IDSToplevel, a choice is made between the exclusive coordinates. + skip_complex: Whether to skip over populating complex numbers. This is + useful for maintaining compatibility with older versions of netCDF4 + (<1.7.0) where complex numbers are not supported. """ if isinstance(structure, IDSToplevel): unsupported_ids_name = ( @@ -218,7 +228,9 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): for child in structure: if isinstance(child, IDSStructure): - exclusive_coordinates.extend(fill_consistent(child, leave_empty)) + exclusive_coordinates.extend( + fill_consistent(child, leave_empty, skip_complex) + ) elif isinstance(child, IDSStructArray): if child.metadata.coordinates[0].references: @@ -230,7 +242,7 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): if isinstance(coor, IDSPrimitive): # maybe fill with random data: try: - maybe_set_random_value(coor, leave_empty) + maybe_set_random_value(coor, leave_empty, skip_complex) except (RuntimeError, ValueError): pass child.resize(len(coor)) @@ -244,7 +256,9 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): else: child.resize(child.metadata.coordinates[0].size or 1) for ele in child: - exclusive_coordinates.extend(fill_consistent(ele, leave_empty)) + exclusive_coordinates.extend( + fill_consistent(ele, leave_empty, skip_complex) + ) else: # IDSPrimitive coordinates = child.metadata.coordinates @@ -256,7 +270,7 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): exclusive_coordinates.append(child) else: try: - maybe_set_random_value(child, leave_empty) + maybe_set_random_value(child, leave_empty, skip_complex) except (RuntimeError, ValueError): pass @@ -278,7 +292,7 @@ def fill_consistent(structure: IDSStructure, leave_empty: float = 0.2): coor = filled_refs.pop() unset_coordinate(coor) - maybe_set_random_value(element, leave_empty) + maybe_set_random_value(element, leave_empty, skip_complex) else: return exclusive_coordinates @@ -301,7 +315,9 @@ def callback(element): visit_children(callback, parent) -def compare_children(st1, st2, deleted_paths=set(), accept_lazy=False): +def compare_children( + st1, st2, deleted_paths=set(), accept_lazy=False, skip_complex=False +): """Perform a deep compare of two structures using asserts. All paths in ``deleted_paths`` are asserted that they are deleted in st2. diff --git a/imaspy/test/test_nc_autofill.py b/imaspy/test/test_nc_autofill.py index e0d3fe91..01280672 100644 --- a/imaspy/test/test_nc_autofill.py +++ b/imaspy/test/test_nc_autofill.py @@ -1,11 +1,52 @@ from imaspy.db_entry import DBEntry +from imaspy.exception import InvalidNetCDFEntry from imaspy.test.test_helpers import compare_children, fill_consistent +import re +import pytest +import netCDF4 +from packaging import version -def test_nc_latest_dd_autofill_put_get(ids_name, tmp_path): +def test_nc_latest_dd_autofill_put_get_skip_complex(ids_name, tmp_path): with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry: ids = entry.factory.new(ids_name) - fill_consistent(ids, 0.5) + fill_consistent(ids, leave_empty=0.5, skip_complex=True) + + entry.put(ids) + ids2 = entry.get(ids_name) + + compare_children(ids, ids2) + + +@pytest.mark.skipif( + version.parse(netCDF4.__version__) < version.parse("1.7.0"), + reason="NetCDF4 versions < 1.7.0 do not support complex numbers", +) +def test_nc_latest_dd_autofill_put_get_with_complex(ids_name, tmp_path): + with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry: + ids = entry.factory.new(ids_name) + fill_consistent(ids, leave_empty=0.5, skip_complex=False) + try: + entry.put(ids) + ids2 = entry.get(ids_name) + compare_children(ids, ids2) + except InvalidNetCDFEntry as e: + # This is expected, as these versions of NetCDF4 do not support + # complex numbers. + if not re.search( + r".*NetCDF 1.7.0 or later is required for complex data types", str(e) + ): + raise InvalidNetCDFEntry(e) from e + + +@pytest.mark.skipif( + version.parse(netCDF4.__version__) >= version.parse("1.7.0"), + reason="NetCDF4 versions >= 1.7.0 support complex numbers", +) +def test_nc_latest_dd_autofill_put_get_with_complex(ids_name, tmp_path): + with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry: + ids = entry.factory.new(ids_name) + fill_consistent(ids, leave_empty=0.5, skip_complex=False) entry.put(ids) ids2 = entry.get(ids_name) diff --git a/pyproject.toml b/pyproject.toml index dccd6912..36e5fffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ docs = [ ] imas-core = [ "imas-core@git+ssh://git@git.iter.org/imas/al-core.git@main" ] netcdf = [ - "netCDF4>=1.7.0", + "netCDF4>=1.4.1", ] h5py = [ "h5py", From 6609cf085576473a032a1c38b8e0fc969f66ac5b Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 14:00:44 +0000 Subject: [PATCH 08/11] Minor changes to tests. --- conftest.py | 5 ----- imaspy/test/test_cli.py | 3 +-- imaspy/test/test_dbentry.py | 3 +-- imaspy/test/test_helpers.py | 4 +--- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index d1893f76..91a9a046 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,6 @@ # - Fixtures that are useful across test modules import functools -import importlib import logging import os import sys @@ -73,10 +72,6 @@ def pytest_addoption(parser): } -# This is a dummy fixture, usually provided by pytest-xdist that isn't available -# in google3. -# The `worker_id` is only used by tests that require IMAS Core which we never -# run try: import pytest_xdist except ImportError: diff --git a/imaspy/test/test_cli.py b/imaspy/test/test_cli.py index 810acda6..d3642410 100644 --- a/imaspy/test/test_cli.py +++ b/imaspy/test/test_cli.py @@ -13,8 +13,7 @@ @pytest.mark.cli -@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") -def test_imaspy_version(): +def test_imaspy_version(requires_imas): runner = CliRunner() result = runner.invoke(print_version) assert result.exit_code == 0 diff --git a/imaspy/test/test_dbentry.py b/imaspy/test/test_dbentry.py index d67fae0d..cb7ebe12 100644 --- a/imaspy/test/test_dbentry.py +++ b/imaspy/test/test_dbentry.py @@ -82,8 +82,7 @@ def test_dbentry_constructor(): assert get_entry_attrs(entry) == (1, 2, 3, 4, None, 6) -@pytest.mark.skipif(not has_imas, reason="Requires IMAS Core.") -def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path): +def test_ignore_unknown_dd_version(monkeypatch, worker_id, tmp_path, requires_imas): entry = open_dbentry(imaspy.ids_defs.MEMORY_BACKEND, "w", worker_id, tmp_path) ids = entry.factory.core_profiles() ids.ids_properties.homogeneous_time = 0 diff --git a/imaspy/test/test_helpers.py b/imaspy/test/test_helpers.py index 8a651d93..0b7e2b43 100644 --- a/imaspy/test/test_helpers.py +++ b/imaspy/test/test_helpers.py @@ -315,9 +315,7 @@ def callback(element): visit_children(callback, parent) -def compare_children( - st1, st2, deleted_paths=set(), accept_lazy=False, skip_complex=False -): +def compare_children(st1, st2, deleted_paths=set(), accept_lazy=False): """Perform a deep compare of two structures using asserts. All paths in ``deleted_paths`` are asserted that they are deleted in st2. From 2a0b3d5132a319ccc64d80c383dbdda92e1778b9 Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 14:03:50 +0000 Subject: [PATCH 09/11] Rename test. --- imaspy/test/test_nc_autofill.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/imaspy/test/test_nc_autofill.py b/imaspy/test/test_nc_autofill.py index 01280672..806caa7a 100644 --- a/imaspy/test/test_nc_autofill.py +++ b/imaspy/test/test_nc_autofill.py @@ -22,7 +22,9 @@ def test_nc_latest_dd_autofill_put_get_skip_complex(ids_name, tmp_path): version.parse(netCDF4.__version__) < version.parse("1.7.0"), reason="NetCDF4 versions < 1.7.0 do not support complex numbers", ) -def test_nc_latest_dd_autofill_put_get_with_complex(ids_name, tmp_path): +def test_nc_latest_dd_autofill_put_get_with_complex_older_netCDF4( + ids_name, tmp_path +): with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry: ids = entry.factory.new(ids_name) fill_consistent(ids, leave_empty=0.5, skip_complex=False) @@ -43,7 +45,9 @@ def test_nc_latest_dd_autofill_put_get_with_complex(ids_name, tmp_path): version.parse(netCDF4.__version__) >= version.parse("1.7.0"), reason="NetCDF4 versions >= 1.7.0 support complex numbers", ) -def test_nc_latest_dd_autofill_put_get_with_complex(ids_name, tmp_path): +def test_nc_latest_dd_autofill_put_get_with_complex_newer_netCDF4( + ids_name, tmp_path +): with DBEntry(f"{tmp_path}/test-{ids_name}.nc", "x") as entry: ids = entry.factory.new(ids_name) fill_consistent(ids, leave_empty=0.5, skip_complex=False) From 96bbe40430f6069e847a1d70bba8802bdced29b9 Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 14:28:48 +0000 Subject: [PATCH 10/11] Update numpy exception version change to 1.25. --- imaspy/test/test_minimal_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imaspy/test/test_minimal_types.py b/imaspy/test/test_minimal_types.py index d4614de5..07a51346 100644 --- a/imaspy/test/test_minimal_types.py +++ b/imaspy/test/test_minimal_types.py @@ -64,7 +64,7 @@ def test_assign_str_1d(minimal, caplog): # Prevent the expected numpy ComplexWarnings from cluttering pytest output @pytest.mark.filterwarnings( "ignore::numpy.ComplexWarning" - if version.parse(np.__version__) < version.parse("2.0.0") + if version.parse(np.__version__) < version.parse("1.25") else "ignore::numpy.exceptions.ComplexWarning" ) @pytest.mark.parametrize("typ, max_dim", [("flt", 6), ("cpx", 6), ("int", 3)]) @@ -94,7 +94,7 @@ def test_assign_numeric_types(minimal, caplog, typ, max_dim): # np allows casting of complex to float or int, but warns: with pytest.warns( np.ComplexWarning - if version.parse(np.__version__) < version.parse("2.0.0") + if version.parse(np.__version__) < version.parse("1.25") else np.exceptions.ComplexWarning ): caplog.clear() From 771897855ba498cf848d1d7a7ffbc486eb96e4ee Mon Sep 17 00:00:00 2001 From: Anushan Fernando Date: Tue, 21 Jan 2025 15:53:59 +0000 Subject: [PATCH 11/11] Fix bug in skip logic of tests. --- imaspy/test/test_nc_autofill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imaspy/test/test_nc_autofill.py b/imaspy/test/test_nc_autofill.py index 806caa7a..9bbc0f1e 100644 --- a/imaspy/test/test_nc_autofill.py +++ b/imaspy/test/test_nc_autofill.py @@ -19,7 +19,7 @@ def test_nc_latest_dd_autofill_put_get_skip_complex(ids_name, tmp_path): @pytest.mark.skipif( - version.parse(netCDF4.__version__) < version.parse("1.7.0"), + version.parse(netCDF4.__version__) >= version.parse("1.7.0"), reason="NetCDF4 versions < 1.7.0 do not support complex numbers", ) def test_nc_latest_dd_autofill_put_get_with_complex_older_netCDF4( @@ -42,7 +42,7 @@ def test_nc_latest_dd_autofill_put_get_with_complex_older_netCDF4( @pytest.mark.skipif( - version.parse(netCDF4.__version__) >= version.parse("1.7.0"), + version.parse(netCDF4.__version__) < version.parse("1.7.0"), reason="NetCDF4 versions >= 1.7.0 support complex numbers", ) def test_nc_latest_dd_autofill_put_get_with_complex_newer_netCDF4(