diff --git a/imap_processing/cdf/cdf_attribute_manager.py b/imap_processing/cdf/cdf_attribute_manager.py index 61f63cd45..ef36a5db5 100644 --- a/imap_processing/cdf/cdf_attribute_manager.py +++ b/imap_processing/cdf/cdf_attribute_manager.py @@ -79,9 +79,9 @@ def __init__(self, data_dir: Path): self._global_attributes = CdfAttributeManager._load_yaml_data( self.source_dir / DEFAULT_GLOBAL_CDF_ATTRS_FILE ) - self._variable_attributes = dict() + self._variable_attributes: dict = {} - def _load_default_global_attr_schema(self) -> dict: + def _load_default_global_attr_schema(self) -> yaml: """ Load the default global schema from the source directory. @@ -96,7 +96,7 @@ def _load_default_global_attr_schema(self) -> dict: # Load the Schema return CdfAttributeManager._load_yaml_data(default_schema_path) - def _load_default_variable_attr_schema(self) -> dict: + def _load_default_variable_attr_schema(self) -> yaml: """ Load the default variable schema from the source directory. @@ -111,7 +111,9 @@ def _load_default_variable_attr_schema(self) -> dict: # Load the Schema return CdfAttributeManager._load_yaml_data(default_schema_path) - def load_global_attributes(self, file_path: str): + # TODO Change Returning Any from function declared to return "dict[Any, Any]" + + def load_global_attributes(self, file_path: str) -> None: """ Update the global attributes property with the attributes from the file. @@ -149,7 +151,7 @@ def add_global_attribute(self, attribute_name: str, attribute_value: str) -> Non self._global_attributes[attribute_name] = attribute_value @staticmethod - def _load_yaml_data(file_path: str | Path) -> dict: + def _load_yaml_data(file_path: str | Path) -> yaml: """ Load a yaml file from the provided path. @@ -222,7 +224,9 @@ def load_variable_attributes(self, file_name: str) -> None: self._variable_attributes.update(var_attrs) - def get_variable_attributes(self, variable_name: str, check_schema=True) -> dict: + def get_variable_attributes( + self, variable_name: str, check_schema: bool = True + ) -> dict: """ Get the attributes for a given variable name. @@ -246,7 +250,8 @@ def get_variable_attributes(self, variable_name: str, check_schema=True) -> dict # Case to handle attributes not in schema if check_schema is False: if variable_name in self._variable_attributes: - return self._variable_attributes[variable_name] + return_dict: dict = self._variable_attributes[variable_name] + return return_dict # TODO: throw an error? return {} diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 6a7801cc8..8ea2edab9 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -6,6 +6,7 @@ """ from pathlib import Path +from typing import Optional from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager @@ -20,7 +21,7 @@ class ImapCdfAttributes(CdfAttributeManager): Source directory. """ - def __init__(self, source_dir=None): + def __init__(self, source_dir: Optional[Path] = None): """ Set the path to the config directory. @@ -34,7 +35,7 @@ def __init__(self, source_dir=None): else: super().__init__(source_dir) - def add_instrument_global_attrs(self, instrument: str): + def add_instrument_global_attrs(self, instrument: str) -> None: """ Add instrument specific global attributes. @@ -46,7 +47,7 @@ def add_instrument_global_attrs(self, instrument: str): # Looks for file named "imap_{instrument}_global_cdf_attrs.yaml" self.load_global_attributes(f"imap_{instrument}_global_cdf_attrs.yaml") - def add_instrument_variable_attrs(self, instrument: str, level: str): + def add_instrument_variable_attrs(self, instrument: str, level: str) -> None: """ Add instrument specific variable attributes. diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index 517cccce7..52d67c6f8 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -12,6 +12,7 @@ from cdflib.xarray.cdf_to_xarray import ISTP_TO_XARRAY_ATTRS import imap_processing +from imap_processing._version import __version__, __version_tuple__ # noqa: F401 logger = logging.getLogger(__name__) @@ -102,7 +103,7 @@ def load_cdf( return dataset -def write_cdf(dataset: xr.Dataset): +def write_cdf(dataset: xr.Dataset) -> xr.Dataset: """ Write the contents of "data" to a CDF file using cdflib.xarray_to_cdf. @@ -120,8 +121,8 @@ def write_cdf(dataset: xr.Dataset): Returns ------- - file_path : pathlib.Path - Path to the file created. + file_path : xr.Dataset + Xr.Dataset to the file created. """ # Create the filename from the global attributes # Logical_source looks like "imap_swe_l2_counts-1min" @@ -158,7 +159,7 @@ def write_cdf(dataset: xr.Dataset): # The Logical_file_id is always the name of the file without the extension dataset.attrs["Logical_file_id"] = file_path.stem # Add the processing version to the dataset attributes - dataset.attrs["ground_software_version"] = imap_processing.__version__ + dataset.attrs["ground_software_version"] = imap_processing._version.__version__ # Convert the xarray object to a CDF xarray_to_cdf( diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 2eb30f031..ba5016f9c 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -24,6 +24,7 @@ import xarray as xr import imap_processing +from imap_processing._version import __version__, __version_tuple__ # noqa: F401 from imap_processing.cdf.utils import load_cdf, write_cdf # TODO: change how we import things and also folder @@ -59,7 +60,7 @@ logger = logging.getLogger(__name__) -def _parse_args(): +def _parse_args() -> argparse.Namespace: """ Parse the command line arguments. @@ -193,7 +194,7 @@ def _parse_args(): return args -def _validate_args(args): +def _validate_args(args: argparse.Namespace) -> None: """ Ensure that the arguments are valid before kicking off the processing. @@ -264,7 +265,7 @@ def __init__( self.version = version self.upload_to_sdc = upload_to_sdc - def download_dependencies(self): + def download_dependencies(self) -> list: """ Download the dependencies for the instrument. @@ -304,7 +305,7 @@ def download_dependencies(self): ) return file_list - def upload_products(self, products: list[str]): + def upload_products(self, products: list[list[Path]]) -> None: """ Upload data products to the IMAP SDC. @@ -321,7 +322,7 @@ def upload_products(self, products: list[str]): imap_data_access.upload(filename) @final - def process(self): + def process(self) -> None: """ Run the processing workflow and cannot be overridden by subclasses. @@ -331,7 +332,7 @@ def process(self): of new products (files). 3. Post-processing actions such as uploading files to the IMAP SDC. """ - logger.info(f"IMAP Processing Version: {imap_processing.__version__}") + logger.info(f"IMAP Processing Version: {imap_processing._version.__version__}") logger.info(f"Processing {self.__class__.__name__} level {self.data_level}") logger.info("Beginning preprocessing (download dependencies)") dependencies = self.pre_processing() @@ -341,7 +342,7 @@ def process(self): self.post_processing(products) logger.info("Processing complete") - def pre_processing(self): + def pre_processing(self) -> list: """ Complete pre-processing. @@ -357,7 +358,7 @@ def pre_processing(self): return self.download_dependencies() @abstractmethod - def do_processing(self, dependencies: list): + def do_processing(self, dependencies: list): # type: ignore[no-untyped-def] """ Abstract method that processes the IMAP processing steps. @@ -376,7 +377,7 @@ def do_processing(self, dependencies: list): """ raise NotImplementedError - def post_processing(self, datasets: list[xr.Dataset]): + def post_processing(self, datasets: list[xr.Dataset]) -> None: """ Complete post-processing. @@ -398,7 +399,7 @@ def post_processing(self, datasets: list[xr.Dataset]): class Codice(ProcessInstrument): """Process CoDICE.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform CoDICE specific processing. @@ -409,8 +410,8 @@ def do_processing(self, dependencies): Returns ------- - list - List of cdf file paths. + dataset : xr.Dataset + Xr.Dataset of cdf file paths. """ print(f"Processing CoDICE {self.data_level}") @@ -439,7 +440,7 @@ def do_processing(self, dependencies): class Glows(ProcessInstrument): """Process GLOWS.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform GLOWS specific processing. @@ -450,8 +451,8 @@ def do_processing(self, dependencies): Returns ------- - products : list - List of products. + datasets : xr.Dataset + Xr.dataset of products. """ print(f"Processing GLOWS {self.data_level}") if self.data_level == "l1a": @@ -477,7 +478,7 @@ def do_processing(self, dependencies): class Hi(ProcessInstrument): """Process IMAP-Hi.""" - def do_processing(self, dependencies: list): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform IMAP-Hi specific processing. @@ -488,8 +489,8 @@ def do_processing(self, dependencies: list): Returns ------- - products : list - List of products. + datasets : xr.Dataset + Xr.Dataset of products. """ print(f"Processing IMAP-Hi {self.data_level}") @@ -517,7 +518,7 @@ def do_processing(self, dependencies: list): class Hit(ProcessInstrument): """Process HIT.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform HIT specific processing. @@ -528,8 +529,8 @@ def do_processing(self, dependencies): Returns ------- - datasets : list - List of datasets. + datasets : xr.Dataset + Xr.Dataset of datasets. """ print(f"Processing HIT {self.data_level}") @@ -551,6 +552,7 @@ def do_processing(self, dependencies): ) # process data and write all processed data to CDF files l1a_dataset = load_cdf(dependencies[0]) + datasets = hit_l1b(l1a_dataset, self.version) return datasets @@ -558,7 +560,7 @@ def do_processing(self, dependencies): class Idex(ProcessInstrument): """Process IDEX.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform IDEX specific processing. @@ -569,10 +571,11 @@ def do_processing(self, dependencies): Returns ------- - list - List of cdf file paths. + datasets : xr.Dataset + Xr.Dataset of cdf file paths. """ print(f"Processing IDEX {self.data_level}") + dataset: xr.Dataset = [] if self.data_level == "l1": if len(dependencies) > 1: @@ -581,14 +584,16 @@ def do_processing(self, dependencies): f"{dependencies}. Expected only one dependency." ) # read CDF file + dataset = PacketParser(dependencies[0], self.version).data return [dataset] + return dataset class Lo(ProcessInstrument): """Process IMAP-Lo.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform IMAP-Lo specific processing. @@ -599,11 +604,11 @@ def do_processing(self, dependencies): Returns ------- - output_files : list - List of output files. + dataset : xr.Dataset + Xr.Dataset of output files. """ print(f"Processing IMAP-Lo {self.data_level}") - + dataset: xr.Dataset = [] if self.data_level == "l1a": # L1A packet / products are 1 to 1. Should only have # one dependency file @@ -630,12 +635,13 @@ def do_processing(self, dependencies): data_dict[dataset.attrs["Logical_source"]] = dataset dataset = lo_l1c.lo_l1c(data_dict, self.version) return [dataset] + return dataset class Mag(ProcessInstrument): """Process MAG.""" - def do_processing(self, dependencies) -> list[Path]: + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform MAG specific processing. @@ -646,8 +652,8 @@ def do_processing(self, dependencies) -> list[Path]: Returns ------- - output_files : list - List of output files. + dataset : xr.Dataset + Xr.Dataset of output files. """ print(f"Processing MAG {self.data_level}") @@ -689,7 +695,7 @@ def do_processing(self, dependencies) -> list[Path]: class Swapi(ProcessInstrument): """Process SWAPI.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform SWAPI specific processing. @@ -700,8 +706,8 @@ def do_processing(self, dependencies): Returns ------- - product : list - List of products. + dataset : xr.Dataset + Xr.Dataset of products. """ print(f"Processing SWAPI {self.data_level}") @@ -719,7 +725,7 @@ def do_processing(self, dependencies): class Swe(ProcessInstrument): """Process SWE.""" - def do_processing(self, dependencies): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform SWE specific processing. @@ -730,11 +736,10 @@ def do_processing(self, dependencies): Returns ------- - list + dataset : xr.Dataset Path to cdf file. """ print(f"Processing SWE {self.data_level}") - if self.data_level == "l1a": if len(dependencies) > 1: raise ValueError( @@ -744,6 +749,7 @@ def do_processing(self, dependencies): dataset = swe_l1a(Path(dependencies[0]), data_version=self.version) # Right now, we only process science data. Therefore, # we expect only one dataset to be returned. + return [dataset] elif self.data_level == "l1b": @@ -758,12 +764,13 @@ def do_processing(self, dependencies): return [dataset] else: print("Did not recognize data level. No processing done.") + return [dataset] class Ultra(ProcessInstrument): """Process IMAP-Ultra.""" - def do_processing(self, dependencies: list): + def do_processing(self, dependencies: list) -> xr.Dataset: """ Perform IMAP-Ultra specific processing. @@ -774,8 +781,8 @@ def do_processing(self, dependencies: list): Returns ------- - product : list - List of products. + datasets : xr.Dataset + Xr.Dataset of products. """ print(f"Processing IMAP-Ultra {self.data_level}") @@ -805,7 +812,7 @@ def do_processing(self, dependencies: list): return datasets -def main(): +def main() -> None: """ Run the processing for a specific instrument & data level. diff --git a/imap_processing/decom.py b/imap_processing/decom.py index e16a9e93b..9517cee19 100644 --- a/imap_processing/decom.py +++ b/imap_processing/decom.py @@ -8,7 +8,7 @@ from space_packet_parser import parser, xtcedef -def decom_packets(packet_file: str, xtce_packet_definition: str): +def decom_packets(packet_file: str, xtce_packet_definition: str) -> list: """ Unpack CCSDS data packet. diff --git a/imap_processing/utils.py b/imap_processing/utils.py index 23b2c1247..cbe4972bd 100644 --- a/imap_processing/utils.py +++ b/imap_processing/utils.py @@ -3,6 +3,7 @@ import collections import dataclasses import logging +from typing import Optional import numpy as np import pandas as pd @@ -16,7 +17,7 @@ logger = logging.getLogger(__name__) -def sort_by_time(packets, time_key): +def sort_by_time(packets: list, time_key: str) -> list: """ Sort packets by specified key. @@ -37,7 +38,7 @@ def sort_by_time(packets, time_key): return sorted_packets -def group_by_apid(packets: list): +def group_by_apid(packets: list) -> dict: """ Group data by apid. @@ -51,7 +52,7 @@ def group_by_apid(packets: list): grouped_packets : dict Grouped data by apid. """ - grouped_packets = collections.defaultdict(list) + grouped_packets: dict[list] = collections.defaultdict(list) for packet in packets: apid = packet.header["PKT_APID"].raw_value grouped_packets.setdefault(apid, []).append(packet) @@ -59,8 +60,11 @@ def group_by_apid(packets: list): def convert_raw_to_eu( - dataset: xr.Dataset, conversion_table_path, packet_name, **read_csv_kwargs -): + dataset: xr.Dataset, + conversion_table_path: str, + packet_name: str, + **read_csv_kwargs: dict, +) -> xr.Dataset: """ Convert raw data to engineering unit. @@ -139,9 +143,9 @@ def convert_raw_to_eu( def create_dataset( packets: list[Packet], - spacecraft_time_key="shcoarse", - include_header=True, - skip_keys=None, + spacecraft_time_key: str = "shcoarse", + include_header: bool = True, + skip_keys: Optional[str] = None, ) -> xr.Dataset: """ Create dataset for each metadata field. @@ -224,7 +228,7 @@ def create_dataset( return dataset -def update_epoch_to_datetime(dataset: xr.Dataset): +def update_epoch_to_datetime(dataset: xr.Dataset) -> xr.Dataset: """ Update epoch in dataset to datetime object. diff --git a/pyproject.toml b/pyproject.toml index 1203e73ed..7e7e9a94a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,4 +119,5 @@ exclude = ['__init__' ] # don't report on objects that match any of these regex [tool.mypy] follow_imports = 'skip' -exclude = ["tests", "docs", "imap_processing"] +exclude = ["tests", "docs", "ultra", "swe", "swapi", "mag", "lo", "idex", "ialirt", "hit", "hi", "glows", "codice" ] +