Merge pull request #170 from cmu-delphi/staging

Staging

Author: dmytrotsko

src/base/migrations/0002_alter_geography_options.py

@@ -0,0 +1,21 @@
+# Generated by Django 5.2.5 on 2025-10-10 20:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="geography",
+            options={
+                "ordering": ["display_order_number"],
+                "verbose_name": "Geography",
+                "verbose_name_plural": "Geographies",
+            },
+        ),
+    ]

src/epiportal/settings.py

@@ -24,7 +24,7 @@
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
 
-APP_VERSION = "1.0.11"
+APP_VERSION = "1.0.13"
 
 
 EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/")

src/indicators/resources.py

@@ -44,7 +44,8 @@ def fix_boolean_fields(row) -> None:
     return row
 
 
-def process_pathogen(row) -> None:
+def process_pathogens(row) -> None:
+    pathogen_ids = []
     if row["Pathogen/\nDisease Area"]:
         pathogens = row["Pathogen/\nDisease Area"].split(",")
         for pathogen in pathogens:
@@ -57,6 +58,21 @@ def process_pathogen(row) -> None:
                     "used_in": "indicators",
                 },
             )
+            pathogen_ids.append(pathogen_obj.id)
+    row["Pathogen/\nDisease Area"] = ",".join(str(el) for el in pathogen_ids)
+
+
+def process_indicator_set(row, source_type="covidcast") -> None:
+    if row["Indicator Set"]:
+        indicator_set_name = row["Indicator Set"].strip()
+        indicator_set_obj, _ = IndicatorSet.objects.get_or_create(
+            name=indicator_set_name,
+            defaults={
+                "name": indicator_set_name,
+                "source_type": source_type,
+            },
+        )
+        row["Indicator Set"] = indicator_set_obj.id
 
 
 def process_indicator_type(row) -> None:
@@ -246,7 +262,7 @@ class IndicatorBaseResource(ModelResource):
     base = Field(
         attribute="base",
         column_name="base",
-        widget=PermissiveForeignKeyWidget(Indicator, field="id"),
+        widget=PermissiveForeignKeyWidget(Indicator),
     )
     source = Field(
         attribute="source",
@@ -281,7 +297,7 @@ class IndicatorResource(ModelResource):
     pathogens = Field(
         attribute="pathogens",
         column_name="Pathogen/\nDisease Area",
-        widget=ManyToManyWidget(Pathogen, field="name", separator=","),
+        widget=ManyToManyWidget(Pathogen),
     )
     indicator_type = Field(
         attribute="indicator_type",
@@ -364,7 +380,7 @@ class IndicatorResource(ModelResource):
     indicator_set = Field(
         attribute="indicator_set",
         column_name="Indicator Set",
-        widget=PermissiveForeignKeyWidget(IndicatorSet, field="name"),
+        widget=PermissiveForeignKeyWidget(IndicatorSet),
     )
 
     class Meta:
@@ -410,20 +426,44 @@ class Meta:
             "restrictions",
             "indicator_set",
         ]
-        import_id_fields: list[str] = ["name", "source"]
+        import_id_fields: list[str] = ["name", "indicator_set", "source"]
         skip_unchanged = True
 
+    def get_instance(self, instance_loader, row):
+        name = row.get("Signal")
+        source = row.get("Source Subdivision")
+        indicator_set = row.get("Indicator Set")
+
+        # Try to match by (name, source)
+        if name and source:
+            try:
+                return self._meta.model.objects.get(name=name, source__id=source)
+            except self._meta.model.DoesNotExist:
+                pass
+
+        # Try to match by (name, indicator_set)
+        if name and indicator_set:
+            try:
+                return self._meta.model.objects.get(
+                    name=name, indicator_set__id=indicator_set
+                )
+            except self._meta.model.DoesNotExist:
+                pass
+
+        return None
+
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
         fix_boolean_fields(row)
-        process_pathogen(row)
+        process_pathogens(row)
         process_indicator_type(row)
         process_format_type(row)
         process_category(row)
         process_geographic_scope(row)
         process_source(row)
         process_severity_pyramid_rungs(row)
         process_available_geographies(row)
+        process_indicator_set(row)
 
     def after_import_row(self, row, row_result, **kwargs):
         process_indicator_geography(row)
@@ -537,7 +577,7 @@ class OtherEndpointIndicatorResource(ModelResource):
     indicator_set = Field(
         attribute="indicator_set",
         column_name="Indicator Set",
-        widget=PermissiveForeignKeyWidget(IndicatorSet, field="name"),
+        widget=PermissiveForeignKeyWidget(IndicatorSet),
     )
 
     class Meta:
@@ -590,13 +630,14 @@ def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
         fix_boolean_fields(row)
         process_source(row)
-        process_pathogen(row)
+        process_pathogens(row)
         process_indicator_type(row)
         process_format_type(row)
         process_category(row)
         process_geographic_scope(row)
         process_severity_pyramid_rungs(row)
         process_available_geographies(row)
+        process_indicator_set(row, source_type="other_endpoint")
 
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:
@@ -619,7 +660,7 @@ class NonDelphiIndicatorResource(resources.ModelResource):
     indicator_set = Field(
         attribute="indicator_set",
         column_name="Indicator Set",
-        widget=PermissiveForeignKeyWidget(NonDelphiIndicatorSet, field="name"),
+        widget=PermissiveForeignKeyWidget(NonDelphiIndicatorSet),
     )
 
     class Meta:
@@ -637,6 +678,7 @@ class Meta:
     def before_import_row(self, row, **kwargs) -> None:
         """Post-processes each row after importing."""
         fix_boolean_fields(row)
+        process_indicator_set(row, source_type="non_delphi")
 
     def skip_row(self, instance, original, row, import_validation_errors=None):
         if not row["Include in indicator app"]:

src/indicatorsets/resources.py

@@ -22,6 +22,7 @@ def process_geographic_scope(row) -> None:
 
 
 def process_severity_pyramid_rungs(row) -> None:
+    severity_pyramid_rung_ids = []
     if row["Surveillance Categories"]:
         severity_pyramid_rungs = row["Surveillance Categories"].split(",")
         for spr in severity_pyramid_rungs:
@@ -31,9 +32,12 @@ def process_severity_pyramid_rungs(row) -> None:
                 used_in="indicatorsets",
                 defaults={"display_name": spr_name.capitalize()},
             )
+            severity_pyramid_rung_ids.append(severity_pyramid_rung_obj.id)
+    row["Surveillance Categories"] = ",".join(map(str, severity_pyramid_rung_ids))
 
 
 def process_pathogens(row) -> None:
+    pathogen_ids = []
     if row["Pathogen(s)/Syndrome(s)"]:
         pathogens = row["Pathogen(s)/Syndrome(s)"].split(",")
         for pathogen in pathogens:
@@ -46,10 +50,13 @@ def process_pathogens(row) -> None:
                     "used_in": "indicatorsets",
                 },
             )
+            pathogen_ids.append(pathogen_obj.id)
+    row["Pathogen(s)/Syndrome(s)"] = ",".join(map(str, pathogen_ids))
 
 
 def process_available_geographies(row) -> None:
     available_geographies = []
+    available_geographies_ids = []
     try:
         if row["Geographic Granularity - Delphi"]:
             available_geographies = row["Geographic Granularity - Delphi"].split(",")
@@ -71,6 +78,9 @@ def process_available_geographies(row) -> None:
             used_in="indicatorsets",
             defaults=default_params,
         )
+        available_geographies_ids.append(geography_obj.id)
+    row["Geographic Levels"] = ",".join(map(str, available_geographies_ids))
+    row["Geographic Granularity - Delphi"] = row["Geographic Levels"]
 
 
 def fix_boolean_fields(row) -> None:
@@ -120,7 +130,7 @@ class IndicatorSetResource(resources.ModelResource):
     pathogens = Field(
         attribute="pathogens",
         column_name="Pathogen(s)/Syndrome(s)",
-        widget=ManyToManyWidget(Pathogen, field="name", separator=","),
+        widget=ManyToManyWidget(Pathogen),
     )
     data_type = Field(attribute="data_type", column_name="Type(s) of Data*")
     geographic_scope = Field(
@@ -131,7 +141,7 @@ class IndicatorSetResource(resources.ModelResource):
     geographic_levels = Field(
         attribute="geographic_levels",
         column_name="Geographic Granularity - Delphi",
-        widget=ManyToManyWidget(Geography, field="name", separator=","),
+        widget=ManyToManyWidget(Geography),
     )
     preprocessing_description = Field(
         attribute="preprocessing_description",
@@ -172,7 +182,7 @@ class IndicatorSetResource(resources.ModelResource):
     severity_pyramid_rungs = Field(
         attribute="severity_pyramid_rungs",
         column_name="Surveillance Categories",
-        widget=ManyToManyWidget(SeverityPyramidRung, field="name", separator=","),
+        widget=ManyToManyWidget(SeverityPyramidRung),
     )
 
     class Meta:
@@ -260,7 +270,7 @@ class NonDelphiIndicatorSetResource(resources.ModelResource):
     pathogens = Field(
         attribute="pathogens",
         column_name="Pathogen(s)/Syndrome(s)",
-        widget=ManyToManyWidget(Pathogen, field="name", separator=","),
+        widget=ManyToManyWidget(Pathogen),
     )
     data_type = Field(attribute="data_type", column_name="Type(s) of Data*")
     geographic_scope = Field(
@@ -271,7 +281,7 @@ class NonDelphiIndicatorSetResource(resources.ModelResource):
     geographic_levels = Field(
         attribute="geographic_levels",
         column_name="Geographic Levels",
-        widget=ManyToManyWidget(Geography, field="name", separator=","),
+        widget=ManyToManyWidget(Geography),
     )
     preprocessing_description = Field(
         attribute="preprocessing_description",
@@ -312,7 +322,7 @@ class NonDelphiIndicatorSetResource(resources.ModelResource):
     severity_pyramid_rungs = Field(
         attribute="severity_pyramid_rungs",
         column_name="Surveillance Categories",
-        widget=ManyToManyWidget(SeverityPyramidRung, field="name", separator=","),
+        widget=ManyToManyWidget(SeverityPyramidRung),
     )
 
     class Meta: