From 5c1524f2a42c1f088238e0aaf38ca83f4b58cc0b Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 15 Dec 2023 17:42:15 -0800 Subject: [PATCH 1/7] School escorting estimation updates Most changes are needed to avoid crash if estimation run actually had no school escorting in the input data --- activitysim/abm/models/school_escorting.py | 132 ++++++++++++------ .../models/util/school_escort_tours_trips.py | 8 ++ 2 files changed, 101 insertions(+), 39 deletions(-) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index b3aaf2b605..2a4cc7babf 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -140,7 +140,7 @@ def add_prev_choices_to_choosers( stage_alts, how="left", left_on=escorting_choice, - right_on=stage_alts.index.name, + right_index=True, ) .set_index("household_id") ) @@ -216,8 +216,12 @@ def create_school_escorting_bundles_table(choosers, tours, stage): bundles : pd.DataFrame one school escorting bundle per row """ - # making a table of bundles - choosers = choosers.reset_index() + # want to keep household_id in columns, which is already there if running in estimation mode + if "household_id" in choosers.columns: + choosers = choosers.reset_index(drop=True) + else: + choosers = choosers.reset_index() + # creating a row for every school escorting bundle choosers = choosers.loc[choosers.index.repeat(choosers["nbundles"])] bundles = pd.DataFrame() @@ -460,7 +464,11 @@ def school_escorting( trace_hh_id = state.settings.trace_hh_id - alts = simulate.read_model_alts(state, model_settings.ALTS, set_index="Alt") + # FIXME setting index as "Alt" causes crash in estimation mode... + # happens in joint_tour_frequency_composition too! + # alts = simulate.read_model_alts(state, model_settings["ALTS"], set_index="Alt") + alts = simulate.read_model_alts(state, model_settings["ALTS"], set_index=None) + alts.index = alts["Alt"].values choosers, participant_columns = determine_escorting_participants( households_merged, persons, model_settings @@ -478,7 +486,11 @@ def school_escorting( for stage_num, stage in enumerate(school_escorting_stages): stage_trace_label = trace_label + "_" + stage estimator = estimation.manager.begin_estimation( - state, "school_escorting_" + stage + state, model_name="school_escorting_" + stage, bundle_name="school_escorting" + ) + + model_spec_raw = simulate.read_model_spec( + file_name=model_settings[stage.upper() + "_SPEC"] ) model_spec_raw = state.filesystem.read_model_spec( @@ -533,9 +545,26 @@ def school_escorting( if estimator: estimator.write_model_settings(model_settings, model_settings_file_name) - estimator.write_spec(model_settings) - estimator.write_coefficients(coefficients_df, model_settings) + estimator.write_spec(model_settings, tag=stage.upper() + "_SPEC") + estimator.write_coefficients( + coefficients_df, file_name=stage.upper() + "_COEFFICIENTS" + ) estimator.write_choosers(choosers) + estimator.write_alternatives(alts, bundle_directory=True) + + # FIXME #interaction_simulate_estimation_requires_chooser_id_in_df_column + # shuold we do it here or have interaction_simulate do it? + # chooser index must be duplicated in column or it will be omitted from interaction_dataset + # estimation requires that chooser_id is either in index or a column of interaction_dataset + # so it can be reformatted (melted) and indexed by chooser_id and alt_id + assert choosers.index.name == "household_id" + assert "household_id" not in choosers.columns + choosers["household_id"] = choosers.index + + # FIXME set_alt_id - do we need this for interaction_simulate estimation bundle tables? + estimator.set_alt_id("alt_id") + + estimator.set_chooser_id(choosers.index.name) log_alt_losers = state.settings.log_alt_losers @@ -580,47 +609,72 @@ def school_escorting( if stage_num >= 1: choosers["Alt"] = choices - choosers = choosers.join(alts, how="left", on="Alt") + choosers = choosers.join(alts.set_index("Alt"), how="left", on="Alt") bundles = create_school_escorting_bundles_table( choosers[choosers["Alt"] > 1], tours, stage ) escort_bundles.append(bundles) escort_bundles = pd.concat(escort_bundles) - escort_bundles["bundle_id"] = ( - escort_bundles["household_id"] * 10 - + escort_bundles.groupby("household_id").cumcount() - + 1 - ) - escort_bundles.sort_values( - by=["household_id", "school_escort_direction"], - ascending=[True, False], - inplace=True, - ) - school_escort_tours = school_escort_tours_trips.create_pure_school_escort_tours( - state, escort_bundles - ) - chauf_tour_id_map = { - v: k for k, v in school_escort_tours["bundle_id"].to_dict().items() - } - escort_bundles["chauf_tour_id"] = np.where( - escort_bundles["escort_type"] == "ride_share", - escort_bundles["first_mand_tour_id"], - escort_bundles["bundle_id"].map(chauf_tour_id_map), - ) - assert ( - escort_bundles["chauf_tour_id"].notnull().all() - ), f"chauf_tour_id is null for {escort_bundles[escort_bundles['chauf_tour_id'].isna()]}. Check availability conditions." + # Only want to create bundles and tours and trips if at least one household has school escorting + if len(escort_bundles) > 0: + escort_bundles["bundle_id"] = ( + escort_bundles["household_id"] * 10 + + escort_bundles.groupby("household_id").cumcount() + + 1 + ) + escort_bundles.sort_values( + by=["household_id", "school_escort_direction"], + ascending=[True, False], + inplace=True, + ) - tours = school_escort_tours_trips.add_pure_escort_tours(tours, school_escort_tours) - tours = school_escort_tours_trips.process_tours_after_escorting_model( - state, escort_bundles, tours - ) + school_escort_tours = school_escort_tours_trips.create_pure_school_escort_tours( + escort_bundles + ) + chauf_tour_id_map = { + v: k for k, v in school_escort_tours["bundle_id"].to_dict().items() + } + escort_bundles["chauf_tour_id"] = np.where( + escort_bundles["escort_type"] == "ride_share", + escort_bundles["first_mand_tour_id"], + escort_bundles["bundle_id"].map(chauf_tour_id_map), + ) - school_escort_trips = school_escort_tours_trips.create_school_escort_trips( - escort_bundles - ) + assert ( + escort_bundles["chauf_tour_id"].notnull().all() + ), f"chauf_tour_id is null for {escort_bundles[escort_bundles['chauf_tour_id'].isna()]}. Check availability conditions." + + tours = school_escort_tours_trips.add_pure_escort_tours( + tours, school_escort_tours + ) + tours = school_escort_tours_trips.process_tours_after_escorting_model( + escort_bundles, tours + ) + + school_escort_trips = school_escort_tours_trips.create_school_escort_trips( + escort_bundles + ) + + else: + # create empty school escort tours & trips tables to be used downstream + tours["school_esc_outbound"] = pd.NA + tours["school_esc_inbound"] = pd.NA + tours["school_escort_direction"] = pd.NA + tours["next_pure_escort_start"] = pd.NA + school_escort_tours = pd.DataFrame(columns=tours.columns) + trip_cols = [ + "household_id", + "person_id", + "tour_id", + "trip_id", + "outbound", + "depart", + "purpose", + "destination", + ] + school_escort_trips = pd.DataFrame(columns=trip_cols) school_escort_trips["primary_purpose"] = school_escort_trips[ "primary_purpose" diff --git a/activitysim/abm/models/util/school_escort_tours_trips.py b/activitysim/abm/models/util/school_escort_tours_trips.py index d8b5386eba..1529669f56 100644 --- a/activitysim/abm/models/util/school_escort_tours_trips.py +++ b/activitysim/abm/models/util/school_escort_tours_trips.py @@ -405,6 +405,10 @@ def merge_school_escort_trips_into_pipeline(state: workflow.State): tours = state.get_dataframe("tours") trips = state.get_dataframe("trips") + # checking to see if there are school escort trips to merge in + if len(school_escort_trips) == 0: + return trips + # want to remove stops if school escorting takes place on that half tour so we can replace them with the actual stops out_se_tours = tours[ tours["school_esc_outbound"].isin(["pure_escort", "ride_share"]) @@ -643,6 +647,10 @@ def force_escortee_tour_modes_to_match_chauffeur(state: workflow.State, tours): # Does it even matter if trip modes are getting matched later? escort_bundles = state.get_dataframe("escort_bundles") + if len(escort_bundles) == 0: + # do not need to do anything if no escorting + return tours + # grabbing the school tour ids for each school escort bundle se_tours = escort_bundles[["school_tour_ids", "chauf_tour_id"]].copy() # merging in chauffeur tour mode From cf0a663e293d6d8eae174b3b7c1c31f0b7a4db38 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 8 Mar 2024 13:48:42 -0800 Subject: [PATCH 2/7] blacken --- activitysim/abm/models/school_escorting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index 2a4cc7babf..7151ba34b9 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -486,7 +486,9 @@ def school_escorting( for stage_num, stage in enumerate(school_escorting_stages): stage_trace_label = trace_label + "_" + stage estimator = estimation.manager.begin_estimation( - state, model_name="school_escorting_" + stage, bundle_name="school_escorting" + state, + model_name="school_escorting_" + stage, + bundle_name="school_escorting", ) model_spec_raw = simulate.read_model_spec( From 4470dd4d8fc7f289183e7b0e8654694934519ae7 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 11 Mar 2024 14:10:42 -0700 Subject: [PATCH 3/7] updating to work with Pydantic and State object --- activitysim/abm/models/school_escorting.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index 7151ba34b9..a1f7de7122 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -466,8 +466,8 @@ def school_escorting( # FIXME setting index as "Alt" causes crash in estimation mode... # happens in joint_tour_frequency_composition too! - # alts = simulate.read_model_alts(state, model_settings["ALTS"], set_index="Alt") - alts = simulate.read_model_alts(state, model_settings["ALTS"], set_index=None) + # alts = simulate.read_model_alts(state, model_settings.ALTS, set_index="Alt") + alts = simulate.read_model_alts(state, model_settings.ALTS, set_index=None) alts.index = alts["Alt"].values choosers, participant_columns = determine_escorting_participants( @@ -491,8 +491,8 @@ def school_escorting( bundle_name="school_escorting", ) - model_spec_raw = simulate.read_model_spec( - file_name=model_settings[stage.upper() + "_SPEC"] + model_spec_raw = state.filesystem.read_model_spec( + file_name=getattr(model_settings, stage.upper() + "_SPEC") ) model_spec_raw = state.filesystem.read_model_spec( @@ -633,7 +633,7 @@ def school_escorting( ) school_escort_tours = school_escort_tours_trips.create_pure_school_escort_tours( - escort_bundles + state, escort_bundles ) chauf_tour_id_map = { v: k for k, v in school_escort_tours["bundle_id"].to_dict().items() @@ -652,9 +652,8 @@ def school_escorting( tours, school_escort_tours ) tours = school_escort_tours_trips.process_tours_after_escorting_model( - escort_bundles, tours + state, escort_bundles, tours ) - school_escort_trips = school_escort_tours_trips.create_school_escort_trips( escort_bundles ) From d9392f3899177f4a91a7275e974a425cc7441ec8 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 18 Mar 2024 09:33:09 -0700 Subject: [PATCH 4/7] adding missed columns necessary for no school escorting --- activitysim/abm/models/school_escorting.py | 2 ++ activitysim/abm/models/util/school_escort_tours_trips.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index a1f7de7122..33848aaec5 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -674,6 +674,8 @@ def school_escorting( "depart", "purpose", "destination", + "escort_participants", + "chauf_tour_id", ] school_escort_trips = pd.DataFrame(columns=trip_cols) diff --git a/activitysim/abm/models/util/school_escort_tours_trips.py b/activitysim/abm/models/util/school_escort_tours_trips.py index 1529669f56..fca6f73c3e 100644 --- a/activitysim/abm/models/util/school_escort_tours_trips.py +++ b/activitysim/abm/models/util/school_escort_tours_trips.py @@ -407,6 +407,9 @@ def merge_school_escort_trips_into_pipeline(state: workflow.State): # checking to see if there are school escort trips to merge in if len(school_escort_trips) == 0: + # if no trips, fill escorting columns with NA + trips[["escort_participants", "school_escort_direction", "school_escort_trip_id",]] = pd.NA + state.replace_table("trips", trips) return trips # want to remove stops if school escorting takes place on that half tour so we can replace them with the actual stops From c4394056409f22b5f916fd8da2ba4088f9ee6185 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 18 Mar 2024 10:05:23 -0700 Subject: [PATCH 5/7] blacken --- activitysim/abm/models/util/school_escort_tours_trips.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/util/school_escort_tours_trips.py b/activitysim/abm/models/util/school_escort_tours_trips.py index fca6f73c3e..bcaeebe961 100644 --- a/activitysim/abm/models/util/school_escort_tours_trips.py +++ b/activitysim/abm/models/util/school_escort_tours_trips.py @@ -408,7 +408,13 @@ def merge_school_escort_trips_into_pipeline(state: workflow.State): # checking to see if there are school escort trips to merge in if len(school_escort_trips) == 0: # if no trips, fill escorting columns with NA - trips[["escort_participants", "school_escort_direction", "school_escort_trip_id",]] = pd.NA + trips[ + [ + "escort_participants", + "school_escort_direction", + "school_escort_trip_id", + ] + ] = pd.NA state.replace_table("trips", trips) return trips From 32ab82534ef7c42adce8880fe132f6647f146e66 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Thu, 28 Mar 2024 09:09:51 -0700 Subject: [PATCH 6/7] handling zero escorting cases --- activitysim/abm/models/school_escorting.py | 10 +++++++++- .../abm/models/util/school_escort_tours_trips.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index 33848aaec5..f559fa99a1 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -58,7 +58,14 @@ def determine_escorting_participants( & (persons.cdap_activity == "M") ] households_with_escortees = escortees["household_id"] - choosers = choosers[choosers.index.isin(households_with_escortees)] + if len(households_with_escortees) == 0: + logger.warning("No households with escortees found!") + else: + tot_households = len(choosers) + choosers = choosers[choosers.index.isin(households_with_escortees)] + logger.info( + f"Proceeding with {len(choosers)} households with escortees out of {tot_households} total households" + ) # can specify different weights to determine chaperones persontype_weight = model_settings.PERSON_WEIGHT @@ -676,6 +683,7 @@ def school_escorting( "destination", "escort_participants", "chauf_tour_id", + "primary_purpose", ] school_escort_trips = pd.DataFrame(columns=trip_cols) diff --git a/activitysim/abm/models/util/school_escort_tours_trips.py b/activitysim/abm/models/util/school_escort_tours_trips.py index bcaeebe961..e22a7198f7 100644 --- a/activitysim/abm/models/util/school_escort_tours_trips.py +++ b/activitysim/abm/models/util/school_escort_tours_trips.py @@ -415,7 +415,7 @@ def merge_school_escort_trips_into_pipeline(state: workflow.State): "school_escort_trip_id", ] ] = pd.NA - state.replace_table("trips", trips) + state.add_table("trips", trips) return trips # want to remove stops if school escorting takes place on that half tour so we can replace them with the actual stops From 94a3a8d37518ec974ac1f1c350e5aadd9c5a2518 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 3 Apr 2024 22:01:55 -0700 Subject: [PATCH 7/7] removing duplicate code --- activitysim/abm/models/school_escorting.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/activitysim/abm/models/school_escorting.py b/activitysim/abm/models/school_escorting.py index f559fa99a1..908113e98f 100644 --- a/activitysim/abm/models/school_escorting.py +++ b/activitysim/abm/models/school_escorting.py @@ -498,10 +498,6 @@ def school_escorting( bundle_name="school_escorting", ) - model_spec_raw = state.filesystem.read_model_spec( - file_name=getattr(model_settings, stage.upper() + "_SPEC") - ) - model_spec_raw = state.filesystem.read_model_spec( file_name=getattr(model_settings, stage.upper() + "_SPEC") )