Updated SEMCOG Example

.github/workflows/core_tests.yml

@@ -180,7 +180,7 @@ jobs:
           - prototype_mtc_extended
           - placeholder_sandag
           - prototype_sandag_xborder
-          - prototype_semcog
+          - example_semcog
           - prototype_mwcog
           - placeholder_multiple_zone
       fail-fast: false

.gitignore

@@ -77,3 +77,4 @@ _test_est
 **/output/
 **/_generated_version.py
 docs/**/_generated
+activitysim/examples/example_semcog/configs_mp/settings.yaml

activitysim/examples/example_manifest.yaml

@@ -488,77 +488,55 @@
       data/final_accessibility.csv
       949994345c9965054be260589fd64079e4032d2e4079e46b9eb9c758a40527c3
 
-- name: prototype_semcog
+- name: example_semcog
   description: 67 zone test example for the SEMCOG region
-  # activitysim create -e prototype_semcog -d test_prototype_semcog
-  # cd test_prototype_semcog
+  # activitysim create -e example_semcog -d test_example_semcog
+  # cd test_example_semcog
   # python simulation.py -c configs -o output -d data
   # cd ..
   include:
-    - prototype_semcog/extensions
-    - prototype_semcog/data
-    - prototype_semcog/configs
-    - prototype_semcog/configs_mp
-    - prototype_semcog/output
-    - prototype_semcog/README.MD
-    - prototype_semcog/simulation.py
+    - example_semcog/extensions
+    - example_semcog/data
+    - example_semcog/configs
+    - example_semcog/configs_mp
+    - example_semcog/output
+    - example_semcog/README.MD
+    - example_semcog/simulation.py
 
-- name: prototype_semcog_z500
-  description: 500 zone intermediate size example for the SEMCOG region
-  # activitysim create -e prototype_semcog_z500 -d test_prototype_semcog_z500
-  # cd test_prototype_semcog_z500
-  # python simulation.py -c configs -o output -d data
-  # cd ..
-  include:
-    - prototype_semcog/extensions
-    - prototype_semcog/data
-    - prototype_semcog/configs
-    - prototype_semcog/configs_mp
-    - prototype_semcog/output
-    - prototype_semcog/README.MD
-    - prototype_semcog/simulation.py
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_z500/skims.omx
-      data/skims.omx
-      7c824032940d1006e731d715bcee8771572e03558eef40ca1ca5f820bfc071ca
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_z500/households.csv
-      data/households.csv
-      93b74d3ad86c4c49222a2427cca6014a12a34c0b46c8a934fc8f849f943b820f
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_z500/persons.csv
-      data/persons.csv
-      3496761cc3b5f9718349ccd3ab97de7db96b8b5eed535a9bf97248fa087161a1
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_z500/land_use.csv
-      data/land_use.csv
-      d4ddb6f5fc7cd844e3e9149d03d32c841c5cb641a77aea9174045ddb85f27a3a
-
-- name: prototype_semcog_full
+- name: example_semcog_full
   description: 2899 zone full size example for the SEMCOG region
-  # activitysim create -e prototype_semcog_full -d test_prototype_semcog_full
-  # cd test_prototype_semcog_full
+  # activitysim create -e example_semcog_full -d test_example_semcog_full
+  # cd test_example_semcog_full
   # python simulation.py -c configs_mp -c configs -o output -d data
   # cd ..
   include:
-    - prototype_semcog/extensions
-    # prototype_semcog/data  # data loaded from activitysim_resources
-    - prototype_semcog/configs
-    - prototype_semcog/configs_mp
-    - prototype_semcog/output
-    - prototype_semcog/README.MD
-    - prototype_semcog/simulation.py
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/skims.omx
-      data/skims.omx
-      efa1f87395df354ccfad7cfe4e67ace96c7ac04fa0290e5e6f8e2593fc0ba6a3
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/skims2.omx
-      data/skims2.omx
-      3cec3453f12c3f07a19ef896b98da2ab79cbcca8bb541f845f6541bf878feeff
+    - example_semcog/extensions
+    - example_semcog/configs
+    - example_semcog/configs_mp
+    - example_semcog/output
+    - example_semcog/README.MD
+    - example_semcog/simulation.py
+    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/skims_offpeak.omx
+      data/skims_offpeak.omx
+      685e93a725e2c04e4b1fc11b92609472c347ce830475448e39b52947fa729d80
+    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/skims_peak.omx
+      data/skims_peak.omx
+      f92d924728c3984bc0d960b0258f6a395be5169167ccc88747503a446ff153f9
     - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/households.csv
       data/households.csv
-      25f5fcabd2953951e0e1cfe3ca2cc8bb4bc63ee11bbdee299a8fc026a775986d
+      34d5db243f7d176c536012f4d475fcc7dc51deb46c070e0e74f5e667578bcd15
     - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/persons.csv
       data/persons.csv
-      934209ad17b789de07ad601c3f77a905a1c35c4bd7c15c62f088c045b3c64f43
+      d8c7eac449efd57758504ca80cf86f150d232b85d659423baab9d203a47dbb6d
     - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/land_use.csv
       data/land_use.csv
-      8f9146993bff315afc6ada52182cb3ea719a6a1ed07095b51535bc7c026285f6
+      2e3c5a8b50d638f6956b3c78c9402ce15f3ceaa4cdc5a51d9d802652b6843772
+    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/maz_to_maz_walk.csv
+      data/maz_to_maz_walk.csv
+      3dd2511cfdac1380653f8cf93a11ee00c625800aad378b46cd1ce191371cec39
+    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/semcog_data_full/maz_to_maz_bike.csv
+      data/maz_to_maz_bike.csv
+      b158a181a48723117c86286a2e3041a35a980143d6b767e3f541c28bfbd38dcf
 
 - name: prototype_arc
   description: 36 zone test example for the ARC region
@@ -893,7 +871,7 @@
     - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/sandag_xborder/traffic_skims_xborder_EV.omx
       data/traffic_skims_xborder_EV.omx
       3826807fbbf5a62b97a70022ea6979cd9d16129e326e5570159221db1cbc4584
-    - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/sandag_xborder/traffic_skims_xborder_MD.omx
+    - cc/media/activitysim/activitysim_resources/master/sandag_xborder/traffic_skims_xborder_MD.omx
       data/traffic_skims_xborder_MD.omx
       9a83cf893e459019328aab726c4bd0fdcd38a2b5ecb88e0aa4a880b6104bbd3a
     - https://media.githubusercontent.com/media/activitysim/activitysim_resources/master/sandag_xborder/traffic_skims_xborder_PM.omx

activitysim/examples/prototype_semcog/configs/accessibility.csv → activitysim/examples/example_semcog/configs/accessibility.csv

@@ -31,6 +31,7 @@ round trip path is available,_rt_available,(_trPkTime_od > 0) & (_trPkTime_do >
 decay function,_decay,_rt_available * exp(_trPkTime * dispersion_parameter_transit)
 transit peak retail,trPkRetail,df.e05_retail * _decay
 transit peak total,trPkTotal,df.tot_emp * _decay
+,trPKHH,df.tot_hhs * _decay
 #,,
 #,, transit off-peak
 #,,

activitysim/examples/prototype_semcog/configs/accessibility.yaml → activitysim/examples/example_semcog/configs/accessibility.yaml

@@ -1,6 +1,6 @@
 
 # columns from land_use table to add to df
-land_use_columns: ['e05_retail', 'tot_emp']
+land_use_columns: ['e05_retail', 'tot_emp', 'tot_hhs']
 
 CONSTANTS:
   # dispersion parameters

activitysim/examples/prototype_semcog/configs/annotate_households.csv → activitysim/examples/example_semcog/configs/annotate_households.csv

@@ -2,7 +2,8 @@ Description,Target,Expression
 #,, annotate households table after import
 ,_PERSON_COUNT,"lambda query, persons, households: persons.query(query).groupby('household_id').size().reindex(households.index).fillna(0).astype(np.int8)"
 #,,FIXME households.income can be negative  - so we clip?
-income,income,households.HINCP.fillna(0)
+#,income,households.hincp * households.adjinc/100000
+income,income,households.hincp.fillna(0)
 income_in_thousands,income_in_thousands,(income / 1000).clip(lower=0)
 income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)"
 #,,
@@ -14,7 +15,7 @@ median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in
 hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)"
 #,,
 #num_workers was renamed in import,,
-,num_workers,"_PERSON_COUNT('(ESR==1)|(ESR==2)|(ESR==4)|(ESR==5)', persons, households)"
+,num_workers,"_PERSON_COUNT('(esr==1)|(esr==2)|(esr==4)|(esr==5)', persons, households)"
 number of non_workers,num_non_workers,households.hhsize - num_workers
 #,,
 #,,we assume that everyone 16 and older is a potential driver
@@ -27,13 +28,15 @@ num_children_6_to_12,num_children_6_to_12,"_PERSON_COUNT('6 <= age <= 12', perso
 num_children_16_to_17,num_children_16_to_17,"_PERSON_COUNT('16 <= age <= 17', persons, households)"
 num_college_age,num_college_age,"_PERSON_COUNT('18 <= age <= 24', persons, households)"
 num_young_adults,num_young_adults,"_PERSON_COUNT('25 <= age <= 34', persons, households)"
-non_family,non_family,households.HHT.isin(HHT_NONFAMILY)
-family,family,households.HHT.isin(HHT_FAMILY)
+non_family,non_family,households.hht.isin(HHT_NONFAMILY)
+family,family,households.hht.isin(HHT_FAMILY)
 home_is_urban,home_is_urban,"reindex(land_use.AreaType, households.home_zone_id) < setting('urban_threshold')"
 home_is_rural,home_is_rural,"reindex(land_use.AreaType, households.home_zone_id) > setting('rural_threshold')"
+home_county,home_county,"reindex(land_use.COUNTY, households.home_zone_id)"
+home_in_detroit,home_in_detroit,"reindex(land_use.COUNTY, households.home_zone_id) == 1"
 #,, default for work and school location logsums before auto_ownership model is run
 ,auto_ownership,households.auto_ownership
-TAZ column to match settings file,TAZ,households.home_zone_id
+TAZ column to match settings file,TAZ,households.TAZ
 number of pre-driving age children in the household,num_predrive_child,"_PERSON_COUNT('ptype == 7', persons, households)"
 number of non-working adult in the household,num_nonworker_adults,"_PERSON_COUNT('ptype == 4', persons, households)"
 number of full time workers,num_fullTime_workers,"_PERSON_COUNT('is_fulltime_worker', persons, households)"

activitysim/examples/prototype_semcog/configs/annotate_landuse.csv → activitysim/examples/example_semcog/configs/annotate_landuse.csv

@@ -3,3 +3,4 @@ Description,Target,Expression
 household_density,household_density,land_use.tot_hhs / (land_use.tot_acres)
 employment_density,employment_density,land_use.tot_emp / (land_use.tot_acres)
 density_index,density_index,(household_density *employment_density) / (household_density + employment_density).clip(lower=1)
+,University_Name,"np.where(land_use.Univ_Tier==1, ""University of Michigan"", "" "")"

activitysim/examples/prototype_semcog/configs/annotate_persons.csv → activitysim/examples/example_semcog/configs/annotate_persons.csv

@@ -7,11 +7,11 @@ age_16_p,age_16_p,persons.age >= 16
 adult,adult,persons.age >= 18
 male,male,persons.sex == 1
 female,female,persons.sex == 2
-,esr,persons.ESR.fillna(0)
-,wkhp,persons.WKHP.fillna(0)
-,wkw,persons.WKW.fillna(0)
-,schg,persons.SCHG.fillna(0)
-,mil,persons.MIL.fillna(0)
+,esr,persons.esr.fillna(0)
+,wkhp,persons.wkhp.fillna(0)
+,wkw,persons.wkw.fillna(0)
+,schg,persons.schg.fillna(0)
+,mil,persons.mil.fillna(0)
 employment status type,pemploy,np.zeros(len(persons))
 ,pemploy,"np.where(persons.age < 16, PEMPLOY_CHILD, PEMPLOY_PART)"
 ,pemploy,"np.where((persons['age'] >= 16) & ((esr == 3) | (esr == 6)), PEMPLOY_NOT, pemploy)"
@@ -60,8 +60,11 @@ is_worker,is_worker,"np.where((pemploy == PEMPLOY_FULL) |( pemploy == PEMPLOY_PA
 home_zone_id,home_zone_id,"reindex(households.home_zone_id, persons.household_id)"
 hh_child,hh_child,"reindex(households.children, persons.household_id)"
 person number,PNUM,persons.member_id
-income,income,"reindex(households.HINCP, persons.household_id)"
+income,income,"reindex(households.hincp, persons.household_id)"
 income_in_thousands,income_in_thousands,(income / 1000).clip(lower=0)
 income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60, 100, np.inf], labels=[1, 2, 3, 4]).astype(int)"
 is_fulltime_worker,is_fulltime_worker,"((age_16_p) & (wkhp >=35) & (wkw>=1) & (wkw<=4) & (~esr.isin([3,6])))"
 is_parttime_worker,is_parttime_worker,"((age_16_p) & (~esr.isin([3,6])) & (is_fulltime_worker == False))"
+recoding existing as legacy,naicsp_recode,"persons['naicsp'].replace('-9', '0000')"
+coding industry according to pums,industry_naics,naicsp_recode.str[0].astype(int)*10
+coding naicsp for telecommuting freq model, naics_category,naicsp_recode.str[0:2]

activitysim/examples/prototype_semcog/configs/annotate_persons_after_hh.csv → activitysim/examples/example_semcog/configs/annotate_persons_after_hh.csv

@@ -3,7 +3,7 @@ Description,Target,Expression
 #,, adults get full hh_value_of_time and children get 60%
 ,_hh_vot,"reindex(households.hh_value_of_time, persons.household_id)"
 ,value_of_time,"_hh_vot.where(persons.age>=18, _hh_vot * 0.667)"
-,_hh_income,"reindex(households.HINCP, persons.household_id)"
+,_hh_income,"reindex(households.hincp, persons.household_id)"
 ,_num_adults,"reindex(households.num_adults, persons.household_id)"
 ,_num_predrive_child,"reindex(households.num_predrive_child, persons.household_id)"
 ,_num_nonworker_adults,"reindex(households.num_nonworker_adults, persons.household_id)"
@@ -20,3 +20,5 @@ Presence of predrive child in HHs,is_pre_drive_child_in_HH,_num_predrive_child>0
 ,_has_children_6_to_12,"reindex(households.num_children_6_to_12, persons.household_id)"
 has_young_children,has_young_children,_has_young_children>0
 has_children_6_to_12,has_children_6_to_12,_has_children_6_to_12>0
+home_county_,home_county_,"reindex(households.home_county, persons.household_id)"
+home_county, home_county, home_county_

activitysim/examples/prototype_semcog/configs/annotate_persons_cdap.csv → activitysim/examples/example_semcog/configs/annotate_persons_cdap.csv

@@ -4,4 +4,4 @@ travel_active,travel_active,persons.cdap_activity != CDAP_ACTIVITY_HOME
 under16_not_at_school,under16_not_at_school,"persons.ptype.isin([PTYPE_SCHOOL, PTYPE_PRESCHOOL]) & persons.cdap_activity.isin(['N', 'H'])"
 has_preschool_kid_at_home,has_preschool_kid_at_home,"other_than(persons.household_id, (persons.ptype == PTYPE_PRESCHOOL) & (persons.cdap_activity == 'H'))"
 has_school_kid_at_home,has_school_kid_at_home,"other_than(persons.household_id, (persons.ptype == PTYPE_SCHOOL) & (persons.cdap_activity == 'H'))"
-,cdap_activity,"np.where((persons.work_from_home) & (persons.cdap_activity=='M'), np.random.choice(['N','H']), persons.cdap_activity)"
+#,cdap_activity,"np.where((persons.work_from_home) & (persons.cdap_activity=='M'), np.random.choice(['N','H']), persons.cdap_activity)"

activitysim/examples/prototype_semcog/configs/annotate_persons_school.csv → activitysim/examples/example_semcog/configs/annotate_persons_school.csv

@@ -5,3 +5,5 @@ Description,Target,Expression
 temp auto_time_to_school,_auto_time_to_school,"skim_dict.lookup(persons.home_zone_id, persons.school_zone_id, ('SOV_TIME', 'MD'))"
 temp auto_time_return,_auto_time_return,"skim_dict.lookup(persons.school_zone_id, persons.home_zone_id, ('SOV_TIME', 'MD'))"
 free flow roundtrip_auto_time_to_school,roundtrip_auto_time_to_school,"np.where(persons.school_zone_id>=0,_auto_time_to_school + _auto_time_return,0)"
+#,,
+#,home_county,"reindex(land_use.COUNTY, persons.home_zone_id)"

activitysim/examples/prototype_semcog/configs/annotate_persons_workplace.csv → activitysim/examples/example_semcog/configs/annotate_persons_workplace.csv

@@ -30,3 +30,8 @@ work_zone_area_type,work_zone_area_type,"reindex(land_use.AreaType, persons.work
 ,work_auto_savings,"np.where(persons.is_worker, _min_work_walk_transit - roundtrip_auto_time_to_work, 0)"
 #,,auto savings over walk or transit capped at 120 and normalized to unity
 ,work_auto_savings_ratio,"(work_auto_savings / 120.0).clip(-1.0, 1.0)"
+,umich_student,df.school_zone_id.isin(land_use[land_use.University_Name=='University of Michigan'].index.values)
+,umich_worker,df.workplace_zone_id.isin(land_use[land_use.University_Name=='University of Michigan'].index.values)
+#,umich_affiliate,"df.get(umich_worker, default=False) | df.get(umich_student, default=False)"
+#,,
+,dest_hourly_peak_parking_cost,"reindex(land_use.parking_hourly, df.workplace_zone_id)"

activitysim/examples/prototype_semcog/configs/atwork_subtour_destination.csv → activitysim/examples/example_semcog/configs/atwork_subtour_destination.csv

@@ -4,8 +4,6 @@ util_dist,util_dist,@_DIST,coef_dist_atwork
 util_dist_squared,util_dist_squared,"@(_DIST).clip(0,20)**2",coef_dist_squared_atwork
 util_dist_cubed,util_dist_cubed,"@(_DIST).clip(0,20)**3",coef_dist_cubed_atwork
 util_dist_logged,util_dist_logged,@(_DIST).apply(np.log1p),coef_dist_logged_atwork
-# util_size_variable_atwork,Size variable atwork,@df['atwork'].apply(np.log1p),coef_size_variable_atwork
-# util_no_attractions_atwork_size_variable_is_0,"No attractions, atwork size variable is 0",atwork==0,coef_no_attractions_atwork_size_variable_is_0
 util_size_variable_atwork,Size variable atwork,@df['size_term'].apply(np.log1p),coef_size_variable_atwork
 util_no_attractions_atwork_size_variable_is_0,"No attractions, atwork size variable is 0",size_term==0,coef_no_attractions_atwork_size_variable_is_0
 util_mode_choice_logsum,Mode choice logsum,mode_choice_logsum,coef_mode_choice_logsum_atwork

activitysim/examples/prototype_semcog/configs/atwork_subtour_destination.yaml → activitysim/examples/example_semcog/configs/atwork_subtour_destination.yaml

@@ -3,6 +3,10 @@ SPEC: atwork_subtour_destination.csv
 SAMPLE_SPEC:  atwork_subtour_destination_sample.csv
 COEFFICIENTS: atwork_subtour_destination_coeffs.csv
 
+SIZE_TERM_SELECTOR: atwork
+SEGMENTS: 
+  - atwork
+ORIG_ZONE_ID: workplace_zone_id
 
 SAMPLE_SIZE: 30
 

activitysim/examples/prototype_semcog/configs/atwork_subtour_destination_sample.csv → activitysim/examples/example_semcog/configs/atwork_subtour_destination_sample.csv

@@ -4,7 +4,5 @@ util_dist,util_dist,@_DIST,coef_dist_atwork
 util_dist_squared,util_dist_squared,"@(_DIST).clip(0,20)**2",coef_dist_squared_atwork
 util_dist_cubed,util_dist_cubed,"@(_DIST).clip(0,20)**3",coef_dist_cubed_atwork
 util_dist_logged,util_dist_logged,@(_DIST).apply(np.log1p),coef_dist_logged_atwork
-# util_size_variable_atwork,Size variable atwork,@df['atwork'].apply(np.log1p),coef_size_variable_atwork
-# util_no_attractions_atwork_size_variable_is_0,"No attractions, atwork size variable is 0",atwork==0,coef_no_attractions_atwork_size_variable_is_0
 util_size_variable_atwork,Size variable atwork,@df['size_term'].apply(np.log1p),coef_size_variable_atwork
 util_no_attractions_atwork_size_variable_is_0,"No attractions, atwork size variable is 0",size_term==0,coef_no_attractions_atwork_size_variable_is_0