From ea25a669d339485c6975569769c85e8ef9f05d25 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 15 Dec 2023 16:30:41 -0800 Subject: [PATCH 1/4] get all disaggregate accessibility values --- .../abm/models/disaggregate_accessibility.py | 22 +++++++++++++++++++ .../abm/tables/disaggregate_accessibility.py | 19 ++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index ab4f9acef7..06164e6b22 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -846,6 +846,10 @@ def compute_disaggregate_accessibility( state.tracing.register_traceable_table(tablename, df) del df + disagg_model_settings = read_disaggregate_accessibility_yaml( + "disaggregate_accessibility.yaml" + ) + # Run location choice logsums = get_disaggregate_logsums( state, @@ -906,4 +910,22 @@ def compute_disaggregate_accessibility( for k, df in logsums.items(): state.add_table(k, df) + # available post-processing + for annotations in disagg_model_settings.get("postprocess_proto_tables", []): + tablename = annotations["tablename"] + df = state.get_dataframe(tablename) + assert df is not None + assert annotations is not None + assign_columns( + df=df, + model_settings={ + **annotations["annotate"], + **disagg_model_settings["suffixes"], + }, + trace_label=tracing.extend_trace_label( + "disaggregate_accessibility.postprocess", tablename + ), + ) + state.add_table(tablename, df) + return diff --git a/activitysim/abm/tables/disaggregate_accessibility.py b/activitysim/abm/tables/disaggregate_accessibility.py index 8ab0e0820d..0372bd6f47 100644 --- a/activitysim/abm/tables/disaggregate_accessibility.py +++ b/activitysim/abm/tables/disaggregate_accessibility.py @@ -172,14 +172,13 @@ def disaggregate_accessibility(state: workflow.State): accessibility_cols = [ x for x in proto_accessibility_df.columns if "accessibility" in x ] + keep_cols = model_settings.get("KEEP_COLS", accessibility_cols) # Parse the merging parameters assert merging_params is not None # Check if already assigned! - if set(accessibility_cols).intersection(persons_merged_df.columns) == set( - accessibility_cols - ): + if set(keep_cols).intersection(persons_merged_df.columns) == set(keep_cols): return # Find the nearest zone (spatially) with accessibilities calculated @@ -211,7 +210,7 @@ def disaggregate_accessibility(state: workflow.State): # because it will get slightly different logsums for households in the same zone. # This is because different destination zones were selected. To resolve, get mean by cols. right_df = ( - proto_accessibility_df.groupby(merge_cols)[accessibility_cols] + proto_accessibility_df.groupby(merge_cols)[keep_cols] .mean() .sort_values(nearest_cols) .reset_index() @@ -244,9 +243,9 @@ def disaggregate_accessibility(state: workflow.State): ) # Predict the nearest person ID and pull the logsums - matched_logsums_df = right_df.loc[clf.predict(x_pop)][ - accessibility_cols - ].reset_index(drop=True) + matched_logsums_df = right_df.loc[clf.predict(x_pop)][keep_cols].reset_index( + drop=True + ) merge_df = pd.concat( [left_df.reset_index(drop=False), matched_logsums_df], axis=1 ).set_index("person_id") @@ -278,9 +277,9 @@ def disaggregate_accessibility(state: workflow.State): # Check that it was correctly left-joined assert all(persons_merged_df[merge_cols] == merge_df[merge_cols]) - assert any(merge_df[accessibility_cols].isnull()) + assert any(merge_df[keep_cols].isnull()) # Inject merged accessibilities so that it can be included in persons_merged function - state.add_table("disaggregate_accessibility", merge_df[accessibility_cols]) + state.add_table("disaggregate_accessibility", merge_df[keep_cols]) - return merge_df[accessibility_cols] + return merge_df[keep_cols] From 1670dc96cb57a85c9e35a2dffc08d0e79b0f92d5 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 8 Mar 2024 15:53:37 -0800 Subject: [PATCH 2/4] updated settings to work with Pydantic --- .../abm/models/disaggregate_accessibility.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 06164e6b22..4b9dc35517 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -164,6 +164,11 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"): """ NEAREST_METHOD: str = "skims" + postprocess_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = [] + """ + List of preprocessor settings to apply to the proto-population tables after generation. + """ + def read_disaggregate_accessibility_yaml( state: workflow.State, file_name @@ -847,7 +852,7 @@ def compute_disaggregate_accessibility( del df disagg_model_settings = read_disaggregate_accessibility_yaml( - "disaggregate_accessibility.yaml" + state, "disaggregate_accessibility.yaml" ) # Run location choice @@ -911,16 +916,16 @@ def compute_disaggregate_accessibility( state.add_table(k, df) # available post-processing - for annotations in disagg_model_settings.get("postprocess_proto_tables", []): - tablename = annotations["tablename"] + for annotations in disagg_model_settings.postprocess_proto_tables: + tablename = annotations.tablename df = state.get_dataframe(tablename) assert df is not None assert annotations is not None assign_columns( df=df, model_settings={ - **annotations["annotate"], - **disagg_model_settings["suffixes"], + **annotations.annotate.dict(), + **disagg_model_settings.suffixes.dict(), }, trace_label=tracing.extend_trace_label( "disaggregate_accessibility.postprocess", tablename From c6803bf88669357b6f111c4bb062c239e9209ada Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 18 Mar 2024 20:15:19 -0700 Subject: [PATCH 3/4] KEEP_COLS setting --- activitysim/abm/models/disaggregate_accessibility.py | 9 +++++++++ activitysim/abm/tables/disaggregate_accessibility.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 4b9dc35517..2fefc889f9 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -154,6 +154,15 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"): procedure work. """ + KEEP_COLS: list[str] | None + """ + Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged + across the group. Initializing the below as NA if not in the auto ownership level, they are skipped + in the groupby mean and the values are correct. + (It's a way to avoid having to update code to reshape the table and introduce new functionality there.) + If none, will keep all of the columns with "accessibility" in the name. + """ + FROM_TEMPLATES: bool = False annotate_proto_tables: list[DisaggregateAccessibilityAnnotateSettings] = [] """ diff --git a/activitysim/abm/tables/disaggregate_accessibility.py b/activitysim/abm/tables/disaggregate_accessibility.py index 0372bd6f47..c030988730 100644 --- a/activitysim/abm/tables/disaggregate_accessibility.py +++ b/activitysim/abm/tables/disaggregate_accessibility.py @@ -172,7 +172,9 @@ def disaggregate_accessibility(state: workflow.State): accessibility_cols = [ x for x in proto_accessibility_df.columns if "accessibility" in x ] - keep_cols = model_settings.get("KEEP_COLS", accessibility_cols) + keep_cols = model_settings.KEEP_COLS + if keep_cols is None: + keep_cols = accessibility_cols # Parse the merging parameters assert merging_params is not None From ab20012b03ce25f1f0404f1c7313b5456839246a Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 27 Mar 2024 14:45:49 -0700 Subject: [PATCH 4/4] keep_cols update and returning tables --- .../abm/models/disaggregate_accessibility.py | 3 ++- .../abm/tables/disaggregate_accessibility.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/activitysim/abm/models/disaggregate_accessibility.py b/activitysim/abm/models/disaggregate_accessibility.py index 2fefc889f9..8d1102743f 100644 --- a/activitysim/abm/models/disaggregate_accessibility.py +++ b/activitysim/abm/models/disaggregate_accessibility.py @@ -154,7 +154,7 @@ class DisaggregateAccessibilitySettings(PydanticReadable, extra="forbid"): procedure work. """ - KEEP_COLS: list[str] | None + KEEP_COLS: list[str] | None = None """ Disaggreate accessibility table is grouped by the "by" cols above and the KEEP_COLS are averaged across the group. Initializing the below as NA if not in the auto ownership level, they are skipped @@ -931,6 +931,7 @@ def compute_disaggregate_accessibility( assert df is not None assert annotations is not None assign_columns( + state, df=df, model_settings={ **annotations.annotate.dict(), diff --git a/activitysim/abm/tables/disaggregate_accessibility.py b/activitysim/abm/tables/disaggregate_accessibility.py index c030988730..7828e1c4c0 100644 --- a/activitysim/abm/tables/disaggregate_accessibility.py +++ b/activitysim/abm/tables/disaggregate_accessibility.py @@ -107,7 +107,7 @@ def maz_centroids(state: workflow.State): @workflow.table -def proto_disaggregate_accessibility(state: workflow.State): +def proto_disaggregate_accessibility(state: workflow.State) -> pd.DataFrame: # Read existing accessibilities, but is not required to enable model compatibility df = input.read_input_table( state, "proto_disaggregate_accessibility", required=False @@ -130,7 +130,7 @@ def proto_disaggregate_accessibility(state: workflow.State): @workflow.table -def disaggregate_accessibility(state: workflow.State): +def disaggregate_accessibility(state: workflow.State) -> pd.DataFrame: """ This step initializes pre-computed disaggregate accessibility and merges it onto the full synthetic population. Function adds merged all disaggregate accessibility tables to the pipeline but returns nothing. @@ -169,12 +169,11 @@ def disaggregate_accessibility(state: workflow.State): ) merging_params = model_settings.MERGE_ON nearest_method = model_settings.NEAREST_METHOD - accessibility_cols = [ - x for x in proto_accessibility_df.columns if "accessibility" in x - ] - keep_cols = model_settings.KEEP_COLS - if keep_cols is None: - keep_cols = accessibility_cols + + if model_settings.KEEP_COLS is None: + keep_cols = [x for x in proto_accessibility_df.columns if "accessibility" in x] + else: + keep_cols = model_settings.KEEP_COLS # Parse the merging parameters assert merging_params is not None