lint

Author: aysim319

claims_hosp/delphi_claims_hosp/backfill.py

@@ -6,23 +6,23 @@
 
 """
 
+import calendar
 import glob
 import os
 import re
 import shutil
 from datetime import datetime, timedelta
 from typing import Union
-import calendar
 
 # third party
 import pandas as pd
-import pytz
 from delphi_utils import GeoMapper
 
 from .config import Config
 
 gmpr = GeoMapper()
 
+
 def store_backfill_file(claims_filepath, _end_date, backfill_dir, logger):
     """
     Store county level backfill data into backfill_dir.
@@ -71,14 +71,14 @@ def store_backfill_file(claims_filepath, _end_date, backfill_dir, logger):
         "state_id": "string"
     })
 
-    filename = "claims_hosp_as_of_%s.parquet"%datetime.strftime(_end_date, "%Y%m%d")
+    filename = "claims_hosp_as_of_%s.parquet" % datetime.strftime(_end_date, "%Y%m%d")
     path = f"{backfill_dir}/{filename}"
 
     # Store intermediate file into the backfill folder
     try:
         backfilldata.to_parquet(path, index=False)
         logger.info("Stored source data in parquet", filename=filename)
-    except:
+    except:  # pylint: disable=W0702
         logger.info("Failed to store source data in parquet")
     return path
 
@@ -108,7 +108,7 @@ def get_file_with_date(files) -> Union[str, None]:
             if pattern:
                 file_month = datetime.strptime(pattern[0], "%Y%m").replace(day=1)
                 end_date = (file_month + timedelta(days=32)).replace(day=1)
-                if issue_date >= file_month and issue_date < end_date:
+                if file_month <= issue_date < end_date:
                     return filename
         return ""
 
@@ -118,7 +118,9 @@ def get_file_with_date(files) -> Union[str, None]:
         logger.info("Issue date has no matching merged files", issue_date=issue_date.strftime("%Y-%m-%d"))
         return
 
-    logger.info("Adding missing date to merged file", issue_date=issue_date, filename=backfill_file, merged_filename=file_name)
+    logger.info(
+        "Adding missing date to merged file", issue_date=issue_date, filename=backfill_file, merged_filename=file_name
+    )
 
     # Start to merge files
     merge_file = f"{file_name.split('.')[0]}_after_merge.parquet"
@@ -168,7 +170,7 @@ def get_date(file_link):
         logger.info("Not enough days, skipping merging", n_file_days=len(date_list))
         return
 
-    logger.info(f"Merging files", start_date=date_list[0], end_date=date_list[-1])
+    logger.info("Merging files", start_date=date_list[0], end_date=date_list[-1])
     # Start to merge files
     pdList = []
     for fn in new_files:

claims_hosp/delphi_claims_hosp/run.py

@@ -103,14 +103,13 @@ def run_module(params, logger=None):
     # Store backfill data
     if params["indicator"].get("generate_backfill_files", True):
         backfill_dir = params["indicator"]["backfill_dir"]
-        backfill_merge_day = params["indicator"]["backfill_merge_day"]
         if custom_run_flag:
-            backfilled_filepath = store_backfill_file(claims_file, dropdate_dt, backfill_dir)
+            backfilled_filepath = store_backfill_file(claims_file, dropdate_dt, backfill_dir, logger)
             merge_existing_backfill_files(backfill_dir, backfilled_filepath, issue_date, logger)
 
         else:
-            merge_backfill_file(backfill_dir, datetime.today())
-            store_backfill_file(claims_file, dropdate_dt, backfill_dir)
+            merge_backfill_file(backfill_dir, datetime.today(), logger)
+            store_backfill_file(claims_file, dropdate_dt, backfill_dir, logger)
 
     # print out information
     logger.info("Loaded params",