Amended time parsing and unit handling

Author: lpilz

xwrf/accessors.py

@@ -6,7 +6,7 @@
     _collapse_time_dim,
     _decode_times,
     _modify_attrs_to_cf,
-    _remove_units_from_bool_arrays,
+    _remove_invalid_units,
 )
 
 
@@ -41,7 +41,7 @@ def postprocess(self, decode_times=True) -> xr.Dataset:
         """
         ds = (
             self.xarray_obj.pipe(_modify_attrs_to_cf)
-            .pipe(_remove_units_from_bool_arrays)
+            .pipe(_remove_invalid_units)
             .pipe(_collapse_time_dim)
         )
         if decode_times:

xwrf/config.yaml

@@ -22,9 +22,13 @@ time_coords:
   - Time
   - time
 
-boolean_units_attrs:
+invalid_units_attrs:
   - '-'
   - flag
+  - '0/1 Flag'
+  - 'whoknows'
+  - 'category'
+  - 'none'
 
 cf_attribute_map:
   ZNW:
@@ -92,3 +96,23 @@ cf_attribute_map:
     units: Pa
   ST:
     units: kelvin
+  SM100255:
+    units: dimensionless
+  SM028100:
+    units: dimensionless
+  SM007028:
+    units: dimensionless
+  SM000007:
+    units: dimensionless
+  SCB_DOM:
+    units: dimensionless
+  SCB_DOM:
+    units: dimensionless
+  COSALPHA_V:
+    units: dimensionless
+  GREENFRAC:
+    units: dimensionless
+  SOILTEMP:
+    units: kelvin
+  RH:
+    units: %

xwrf/io_plugin.py

@@ -10,21 +10,24 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset:
     """
     Decode the time variable to datetime64.
     """
+    try:
+        _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S')
+    except ValueError:
+        _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%dT%H:%M:%S.%f')
     ds = ds.assign_coords(
         {
-            'Time': pd.to_datetime(
-                ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S'
-            )
+            'Time': _time
         }
     )
     ds.Time.attrs = {'long_name': 'Time', 'standard_name': 'time'}
     return ds
 
 
-def _remove_units_from_bool_arrays(ds: xr.Dataset) -> xr.Dataset:
-    boolean_units_attrs = config.get('boolean_units_attrs')
+def _remove_invalid_units(ds: xr.Dataset) -> xr.Dataset:
+    invalid_units_attrs = config.get('invalid_units_attrs')
+    print(invalid_units_attrs)
     for variable in ds.data_vars:
-        if ds[variable].attrs.get('units') in boolean_units_attrs:
+        if ds[variable].attrs.get('units') in invalid_units_attrs:
             ds[variable].attrs.pop('units', None)
     return ds