Merge pull request #581 from dhensle/flexible_ids

Flexible Number of Tour & Trip IDs

Author: jpn--

activitysim/abm/models/util/canonical_ids.py

@@ -4,8 +4,12 @@
 
 import numpy as np
 import pandas as pd
+import re
 
 from activitysim.core.util import reindex
+from activitysim.core import config
+from activitysim.core import pipeline
+from activitysim.core import simulate
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +58,158 @@ def enumerate_tour_types(tour_flavors):
     return channels
 
 
+def read_alts_file(file_name, set_index=None):
+    try:
+        alts = simulate.read_model_alts(file_name, set_index=set_index)
+    except RuntimeError:
+        logger.warning(f"Could not find file {file_name} to determine tour flavors.")
+        return pd.DataFrame()
+    return alts
+
+
+def parse_tour_flavor_from_columns(columns, tour_flavor):
+    """
+    determines the max number from columns if column name contains tour flavor
+    example: columns={'work1', 'work2'} -> 2
+
+    Parameters
+    ----------
+    columns : list of str
+    tour_flavor : str
+        string subset that you want to find in columns
+
+    Returns
+    -------
+    int
+        max int found in columns with tour_flavor
+    """
+    # below produces a list of numbers present in each column containing the tour flavor string
+    tour_numbers = [(re.findall(r"\d+", col)) for col in columns if tour_flavor in col]
+
+    # flatten list
+    tour_numbers = [int(item) for sublist in tour_numbers for item in sublist]
+
+    # find max
+    try:
+        max_tour_flavor = max(tour_numbers)
+        return max_tour_flavor
+    except ValueError:
+        # could not find a maximum integer for this flavor in the columns
+        return -1
+
+
+def determine_mandatory_tour_flavors(mtf_settings, model_spec, default_flavors):
+    provided_flavors = mtf_settings.get("MANDATORY_TOUR_FLAVORS", None)
+
+    mandatory_tour_flavors = {
+        # hard code work and school tours
+        "work": parse_tour_flavor_from_columns(model_spec.columns, "work"),
+        "school": parse_tour_flavor_from_columns(model_spec.columns, "school"),
+    }
+
+    valid_flavors = (mandatory_tour_flavors["work"] >= 1) & (
+        mandatory_tour_flavors["school"] >= 1
+    )
+
+    if provided_flavors is not None:
+        if mandatory_tour_flavors != provided_flavors:
+            logger.warning(
+                "Specified tour flavors do not match alternative file flavors"
+            )
+            logger.warning(
+                f"{provided_flavors} does not equal {mandatory_tour_flavors}"
+            )
+        # use provided flavors if provided
+        return provided_flavors
+
+    if not valid_flavors:
+        # if flavors could not be parsed correctly and no flavors provided, return the default
+        logger.warning(
+            "Could not determine alts from alt file and no flavors were provided."
+        )
+        logger.warning(f"Using defaults: {default_flavors}")
+        return default_flavors
+
+    return mandatory_tour_flavors
+
+
+def determine_non_mandatory_tour_max_extension(
+    model_settings, extension_probs, default_max_extension=2
+):
+    provided_max_extension = model_settings.get("MAX_EXTENSION", None)
+
+    max_extension = parse_tour_flavor_from_columns(extension_probs.columns, "tour")
+
+    if provided_max_extension is not None:
+        if provided_max_extension != max_extension:
+            logger.warning(
+                "Specified non mandatory tour extension does not match extension probabilities file"
+            )
+        return provided_max_extension
+
+    if (max_extension >= 0) & isinstance(max_extension, int):
+        return max_extension
+
+    return default_max_extension
+
+
+def determine_flavors_from_alts_file(
+    alts, provided_flavors, default_flavors, max_extension=0
+):
+    """
+    determines the max number from alts for each column containing numbers
+    example: alts={'index': ['alt1', 'alt2'], 'escort': [1, 2], 'othdisc': [3, 4]}
+             yelds -> {'escort': 2, 'othdisc': 4}
+
+    will return provided flavors if available
+    else, return default flavors if alts can't be groked
+
+    Parameters
+    ----------
+    alts : pd.DataFrame
+    provided_flavors : dict
+        tour flavors provided by user in the model yaml
+    default_flavors : dict
+        default tour flavors to fall back on
+    max_extension : int
+        scale to increase number of tours accross all alternatives
+
+    Returns
+    -------
+    dict
+        tour flavors
+    """
+    try:
+        flavors = {
+            c: int(alts[c].max() + max_extension)
+            for c in alts.columns
+            if all(alts[c].astype(str).str.isnumeric())
+        }
+        valid_flavors = all(
+            [(isinstance(flavor, str) & (num >= 0)) for flavor, num in flavors.items()]
+        ) & (len(flavors) > 0)
+    except (ValueError, AttributeError):
+        valid_flavors = False
+
+    if provided_flavors is not None:
+        if flavors != provided_flavors:
+            logger.warning(
+                f"Specified tour flavors {provided_flavors} do not match alternative file flavors {flavors}"
+            )
+        # use provided flavors if provided
+        return provided_flavors
+
+    if not valid_flavors:
+        # if flavors could not be parsed correctly and no flavors provided, return the default
+        logger.warning(
+            "Could not determine alts from alt file and no flavors were provided."
+        )
+        logger.warning(f"Using defaults: {default_flavors}")
+        return default_flavors
+
+    return flavors
+
+
 def canonical_tours():
     """
         create labels for every the possible tour by combining tour_type/tour_num.
@@ -63,47 +219,100 @@ def canonical_tours():
         list of canonical tour labels in alphabetical order
     """
 
-    # FIXME we pathalogically know what the possible tour_types and their max tour_nums are
-    # FIXME instead, should get flavors from alts tables (but we would have to know their names...)
-    # alts = pipeline.get_table('non_mandatory_tour_frequency_alts')
-    # non_mandatory_tour_flavors = {c : alts[c].max() for c in alts.columns}
-
-    # - non_mandatory_channels
-    MAX_EXTENSION = 2
-    non_mandatory_tour_flavors = {
-        "escort": 2 + MAX_EXTENSION,
-        "shopping": 1 + MAX_EXTENSION,
-        "othmaint": 1 + MAX_EXTENSION,
-        "othdiscr": 1 + MAX_EXTENSION,
-        "eatout": 1 + MAX_EXTENSION,
-        "social": 1 + MAX_EXTENSION,
+    # ---- non_mandatory_channels
+    nm_model_settings_file_name = "non_mandatory_tour_frequency.yaml"
+    nm_model_settings = config.read_model_settings(nm_model_settings_file_name)
+    nm_alts = read_alts_file("non_mandatory_tour_frequency_alternatives.csv")
+
+    # first need to determine max extension
+    try:
+        ext_probs_f = config.config_file_path(
+            "non_mandatory_tour_frequency_extension_probs.csv"
+        )
+        extension_probs = pd.read_csv(ext_probs_f, comment="#")
+    except RuntimeError:
+        logger.warning(
+            f"non_mandatory_tour_frequency_extension_probs.csv file not found"
+        )
+        extension_probs = pd.DataFrame()
+    max_extension = determine_non_mandatory_tour_max_extension(
+        nm_model_settings, extension_probs, default_max_extension=2
+    )
+
+    provided_nm_tour_flavors = nm_model_settings.get("NON_MANDATORY_TOUR_FLAVORS", None)
+    default_nm_tour_flavors = {
+        "escort": 2 + max_extension,
+        "shopping": 1 + max_extension,
+        "othmaint": 1 + max_extension,
+        "othdiscr": 1 + max_extension,
+        "eatout": 1 + max_extension,
+        "social": 1 + max_extension,
     }
+
+    non_mandatory_tour_flavors = determine_flavors_from_alts_file(
+        nm_alts, provided_nm_tour_flavors, default_nm_tour_flavors, max_extension
+    )
+    # FIXME additional non-mandatory tour flavors are added in school escorting PR
     non_mandatory_channels = enumerate_tour_types(non_mandatory_tour_flavors)
 
-    # - mandatory_channels
-    mandatory_tour_flavors = {"work": 2, "school": 2}
+    logger.info(f"Non-Mandatory tour flavors used are {non_mandatory_tour_flavors}")
+
+    # ---- mandatory_channels
+    mtf_model_settings_file_name = "mandatory_tour_frequency.yaml"
+    mtf_model_settings = config.read_model_settings(mtf_model_settings_file_name)
+    mtf_spec = mtf_model_settings.get("SPEC", "mandatory_tour_frequency.csv")
+    mtf_model_spec = read_alts_file(file_name=mtf_spec)
+    default_mandatory_tour_flavors = {"work": 2, "school": 2}
+
+    mandatory_tour_flavors = determine_mandatory_tour_flavors(
+        mtf_model_settings, mtf_model_spec, default_mandatory_tour_flavors
+    )
     mandatory_channels = enumerate_tour_types(mandatory_tour_flavors)
 
-    # - atwork_subtour_channels
+    logger.info(f"Mandatory tour flavors used are {mandatory_tour_flavors}")
+
+    # ---- atwork_subtour_channels
+    atwork_model_settings_file_name = "atwork_subtour_frequency.yaml"
+    atwork_model_settings = config.read_model_settings(atwork_model_settings_file_name)
+    atwork_alts = read_alts_file("atwork_subtour_frequency_alternatives.csv")
+
+    provided_atwork_flavors = atwork_model_settings.get("ATWORK_SUBTOUR_FLAVORS", None)
+    default_atwork_flavors = {"eat": 1, "business": 2, "maint": 1}
+
+    atwork_subtour_flavors = determine_flavors_from_alts_file(
+        atwork_alts, provided_atwork_flavors, default_atwork_flavors
+    )
+    atwork_subtour_channels = enumerate_tour_types(atwork_subtour_flavors)
+
+    logger.info(f"Atwork subtour flavors used are {atwork_subtour_flavors}")
+
     # we need to distinguish between subtours of different work tours
     # (e.g. eat1_1 is eat subtour for parent work tour 1 and eat1_2 is for work tour 2)
-    atwork_subtour_flavors = {"eat": 1, "business": 2, "maint": 1}
-    atwork_subtour_channels = enumerate_tour_types(atwork_subtour_flavors)
     max_work_tours = mandatory_tour_flavors["work"]
     atwork_subtour_channels = [
         "%s_%s" % (c, i + 1)
         for c in atwork_subtour_channels
         for i in range(max_work_tours)
     ]
 
-    # - joint_tour_channels
-    joint_tour_flavors = {
+    # ---- joint_tour_channels
+    jtf_model_settings_file_name = "joint_tour_frequency.yaml"
+    jtf_model_settings = config.read_model_settings(jtf_model_settings_file_name)
+    jtf_alts = read_alts_file("joint_tour_frequency_alternatives.csv")
+    provided_joint_flavors = jtf_model_settings.get("JOINT_TOUR_FLAVORS", None)
+
+    default_joint_flavors = {
         "shopping": 2,
         "othmaint": 2,
         "othdiscr": 2,
         "eatout": 2,
         "social": 2,
     }
+    joint_tour_flavors = determine_flavors_from_alts_file(
+        jtf_alts, provided_joint_flavors, default_joint_flavors
+    )
+    logger.info(f"Joint tour flavors used are {joint_tour_flavors}")
+
     joint_tour_channels = enumerate_tour_types(joint_tour_flavors)
     joint_tour_channels = ["j_%s" % c for c in joint_tour_channels]
 
@@ -182,14 +391,51 @@ def set_tour_index(tours, parent_tour_num_col=None, is_joint=False):
     return tours
 
 
-def set_trip_index(trips, tour_id_column="tour_id"):
+def determine_max_trips_per_leg(default_max_trips_per_leg=4):
+    model_settings_file_name = "stop_frequency.yaml"
+    model_settings = config.read_model_settings(model_settings_file_name)
+
+    # first see if flavors given explicitly
+    provided_max_trips_per_leg = model_settings.get("MAX_TRIPS_PER_LEG", None)
+
+    # determine flavors from alternative file
+    try:
+        alts = read_alts_file("stop_frequency_alternatives.csv")
+        trips_per_leg = [
+            int(alts[c].max())
+            for c in alts.columns
+            if all(alts[c].astype(str).str.isnumeric())
+        ]
+        max_trips_per_leg = (
+            max(trips_per_leg) + 1
+        )  # adding one for additional trip home or to primary dest
+        if max_trips_per_leg > 1:
+            valid_max_trips = True
+    except (ValueError, RuntimeError):
+        valid_max_trips = False
+
+    if provided_max_trips_per_leg is not None:
+        if provided_max_trips_per_leg != max_trips_per_leg:
+            logger.warning(
+                "Provided max number of stops on tour does not match with stop frequency alternatives file"
+            )
+        return provided_max_trips_per_leg
 
-    MAX_TRIPS_PER_LEG = 4  # max number of trips per leg (inbound or outbound) of tour
+    if valid_max_trips:
+        return max_trips_per_leg
+
+    return default_max_trips_per_leg
+
+
+def set_trip_index(trips, tour_id_column="tour_id"):
+    # max number of trips per leg (inbound or outbound) of tour
+    #  = stops + 1 for primary half-tour destination
+    max_trips_per_leg = determine_max_trips_per_leg()
 
-    # canonical_trip_num: 1st trip out = 1, 2nd trip out = 2, 1st in = 5, etc.
-    canonical_trip_num = (~trips.outbound * MAX_TRIPS_PER_LEG) + trips.trip_num
+    # canonical_trip_num: 1st trip out = 1, 2nd trip out = 2, 1st in = max_trips_per_leg + 1, etc.
+    canonical_trip_num = (~trips.outbound * max_trips_per_leg) + trips.trip_num
     trips["trip_id"] = (
-        trips[tour_id_column] * (2 * MAX_TRIPS_PER_LEG) + canonical_trip_num
+        trips[tour_id_column] * (2 * max_trips_per_leg) + canonical_trip_num
     )
     trips.set_index("trip_id", inplace=True, verify_integrity=True)
 

activitysim/abm/models/util/test/configs/atwork_subtour_frequency_alternatives.csv

@@ -0,0 +1,8 @@
+#,,,alt file for building tours even though simulation is simple_simulate not interaction_simulate
+alt,eat,business,maint
+no_subtours,0,0,0
+eat,1,0,0
+business1,0,1,0
+maint,0,0,1
+business2,0,2,0
+eat_business,1,1,0

activitysim/abm/models/util/test/configs/joint_tour_frequency_alternatives.csv

@@ -0,0 +1,23 @@
+#,,,,,alt file for building joint tours
+alt,shopping,othmaint,eatout,social,othdiscr
+0_tours,0,0,0,0,0
+1_Shop,1,0,0,0,0
+1_Main,0,1,0,0,0
+1_Eat,0,0,1,0,0
+1_Visit,0,0,0,1,0
+1_Disc,0,0,0,0,1
+2_SS,2,0,0,0,0
+2_SM,1,1,0,0,0
+2_SE,1,0,1,0,0
+2_SV,1,0,0,1,0
+2_SD,1,0,0,0,1
+2_MM,0,2,0,0,0
+2_ME,0,1,1,0,0
+2_MV,0,1,0,1,0
+2_MD,0,1,0,0,1
+2_EE,0,0,2,0,0
+2_EV,0,0,1,1,0
+2_ED,0,0,1,0,1
+2_VV,0,0,0,2,0
+2_VD,0,0,0,1,1
+2_DD,0,0,0,0,2