Merge pull request #176 from cmu-delphi/staging

Staging

Author: dmytrotsko

src/datasources/resources.py

@@ -4,6 +4,13 @@
 from datasources.models import SourceSubdivision, OtherEndpointSourceSubdivision
 
 
+def strip_all_string_values(row) -> None:
+    for key, value in row.items():
+        # Check if the value is a string and not None
+        if isinstance(value, str):
+            row[key] = value.strip()
+
+
 class SourceSubdivisionResource(resources.ModelResource):
     name = Field(
         attribute="name",
@@ -68,6 +75,9 @@ class Meta:
         )
         exclude = ("id", )
 
+    def before_import_row(self, row, **kwargs):
+        strip_all_string_values(row)
+
     def after_save_instance(self, instance, row, **kwargs):
         instance.source_type = "other_endpoint"
         instance.save()

src/indicators/resources.py

@@ -62,17 +62,16 @@ def process_pathogens(row) -> None:
     row["Pathogen/\nDisease Area"] = ",".join(str(el) for el in pathogen_ids)
 
 
-def process_indicator_set(row, source_type="covidcast") -> None:
+def process_indicator_set(row) -> None:
     if row["Indicator Set"]:
         indicator_set_name = row["Indicator Set"].strip()
-        indicator_set_obj, _ = IndicatorSet.objects.get_or_create(
-            name=indicator_set_name,
-            defaults={
-                "name": indicator_set_name,
-                "source_type": source_type,
-            },
+        indicator_set_obj = IndicatorSet.objects.get(
+            name=indicator_set_name
         )
-        row["Indicator Set"] = indicator_set_obj.id
+        if indicator_set_obj:
+            row["Indicator Set"] = indicator_set_obj.id
+        else:
+            row["Indicator Set"] = None
 
 
 def process_indicator_type(row) -> None:
@@ -247,6 +246,13 @@ def import_row(self, row, instance_loader, **kwargs):
         return import_result
 
 
+def strip_all_string_values(row) -> None:
+    for key, value in row.items():
+        # Check if the value is a string and not None
+        if isinstance(value, str):
+            row[key] = value.strip()
+
+
 class PermissiveForeignKeyWidget(ForeignKeyWidget):
 
     def clean(self, value, row=None, **kwargs):
@@ -277,6 +283,7 @@ class Meta:
 
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
+        strip_all_string_values(row)
         process_base(row)
 
 
@@ -454,6 +461,7 @@ def get_instance(self, instance_loader, row):
 
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
+        strip_all_string_values(row)
         fix_boolean_fields(row)
         process_pathogens(row)
         process_indicator_type(row)
@@ -475,6 +483,8 @@ def after_save_instance(self, instance, row, **kwargs):
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:
             return True
+        if row["Indicator Set"] is None:
+            return True
 
 
 class OtherEndpointIndicatorResource(ModelResource):
@@ -628,6 +638,7 @@ class Meta:
 
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
+        strip_all_string_values(row)
         fix_boolean_fields(row)
         process_source(row)
         process_pathogens(row)
@@ -637,11 +648,13 @@ def before_import_row(self, row, **kwargs) -> None:
         process_geographic_scope(row)
         process_severity_pyramid_rungs(row)
         process_available_geographies(row)
-        process_indicator_set(row, source_type="other_endpoint")
+        process_indicator_set(row)
 
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:
             return True
+        if row["Indicator Set"] is None:
+            return True
 
     def after_import_row(self, row, row_result, **kwargs):
         process_indicator_geography(row)
@@ -677,8 +690,9 @@ class Meta:
 
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
+        strip_all_string_values(row)
         fix_boolean_fields(row)
-        process_indicator_set(row, source_type="non_delphi")
+        process_indicator_set(row)
 
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:

src/indicatorsets/resources.py

@@ -98,6 +98,13 @@ def fix_boolean_fields(row) -> None:
     return row
 
 
+def strip_all_string_values(row) -> None:
+    for key, value in row.items():
+        # Check if the value is a string and not None
+        if isinstance(value, str):
+            row[key] = value.strip()
+
+
 class IndicatorSetResource(resources.ModelResource):
     name = Field(attribute="name", column_name="Indicator Set name* ")
     short_name = Field(attribute="short_name", column_name="Indicator Set Short Name")
@@ -241,14 +248,19 @@ def skip_row(self, instance, original, row, import_validation_errors=None):
             return True
 
     def before_import_row(self, row, **kwargs):
+        strip_all_string_values(row)
         fix_boolean_fields(row)
         process_geographic_scope(row)
         process_severity_pyramid_rungs(row)
         process_pathogens(row)
         process_available_geographies(row)
 
     def after_save_instance(self, instance, row, **kwargs):
-        instance.source_type = "covidcast" if instance.epidata_endpoint == "covidcast" else "other_endpoint"
+        instance.source_type = (
+            "covidcast"
+            if instance.epidata_endpoint == "covidcast"
+            else "other_endpoint"
+        )
         instance.save()
 
 
@@ -381,6 +393,7 @@ def skip_row(self, instance, original, row, import_validation_errors=None):
             return True
 
     def before_import_row(self, row, **kwargs):
+        strip_all_string_values(row)
         fix_boolean_fields(row)
         process_geographic_scope(row)
         process_severity_pyramid_rungs(row)