Skip to content

Commit

Permalink
Python reformattings (#166)
Browse files Browse the repository at this point in the history 
* inital Draft of python formatting files
* uncontroversial python formatting changes
* renaming and other unproblematic refactorings for python
* removed duplicate key in dictionary
* refactored the area tree to immediately throw errors and split it into two functions
* moved functionality from the compilation into dedicated modules
* audited instances, where bandit thought we were pawnable and added annotations why we are not
* extracted the prop computation and the arch-name inference to their functions
* minor misc refactorings to python code
* fixes for mypy to stop complaining about imports
* moved code in python files around, to make methods easier to read

Co-authored-by: octycs <[email protected]>
  • Loading branch information
@CommanderStorm @octycs
CommanderStorm and octycs authored Jul 29, 2022
1 parent 3c891bc commit 19f9240
Show file tree
Hide file tree
Showing 22 changed files with 1,422 additions and 1,273 deletions.
165 changes: 34 additions & 131 deletions data/compile.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,27 @@
import logging
import os
import json

from utils import convert_to_webp
from processors import areatree, images, maps, coords, merge, patch, roomfinder, search, sections, sitemap, structure, tumonline

DEBUG_MODE = "GIT_COMMIT_SHA" not in os.environ.keys()


from processors import (
areatree,
coords,
export,
images,
maps,
merge,
roomfinder,
search,
sections,
sitemap,
structure,
tumonline,
)

DEBUG_MODE = "GIT_COMMIT_SHA" not in os.environ


# pylint: disable=too-many-locals
def main():
"""Main function"""
# --- Read base data ---
logging.info("-- 00 areatree")
data = areatree.read_areatree()
Expand All @@ -18,7 +31,7 @@ def main():

logging.info("-- 02 rooms extendend")
data = merge.merge_yaml(data, "sources/02_rooms-extended.yaml")

# Add source information for these entries, which are up to here
# always declared by navigatum
for _id, entry in data.items():
Expand Down Expand Up @@ -51,40 +64,21 @@ def main():

# TODO: Does it make sense to introduce a type 'sub_building' here?

for _id, _data in data.items():
_data["type_common_name"] = {
"root": "Standortübersicht",
"site": "Standort",
"campus": "Campus",
"area": "Gebiet / Gruppe von Gebäuden",
"joined_building": "Gebäudekomplex",
"building": "Gebäudeteil"
if (_data["type"] == "building" and
data[_data["parents"][-1]]["type"] == "joined_building")
else "Gebäude",
"room": _data["usage"]["name"] if "usage" in _data else "Raum",
"virtual_room": _data["usage"]["name"] if "usage" in _data else "Raum/Gebäudeteil",
}[_data["type"]]
logging.info("-- 35 Infer the common_name for every type")
structure.infer_type_common_name(data)

logging.info("-- 40 Coordinates")
coords.assert_buildings_have_coords(data)
coords.assign_coordinates(data)
coords.check_coords(data)
coords.add_and_check_coords(data)

logging.info("-- 45 Roomfinder maps")
maps.assign_roomfinder_maps(data)
maps.remove_non_covering_maps(data)
maps.assign_default_roomfinder_map(data)
maps.build_roomfinder_maps(data)
maps.roomfinder_maps(data)

logging.info("-- 46 Overlay maps")
maps.add_overlay_maps(data)

logging.info(f"-- 50 convert {images.IMAGE_BASE} to webp")
convert_to_webp(images.IMAGE_BASE)
logging.info("-- 51 resize and crop the images for different resolutions and formats")
logging.info("-- 50 Convert, resize and crop the images for different resolutions and formats")
images.resize_and_crop()
logging.info("-- 52 Add image information")
logging.info("-- 51 Add image information")
images.add_img(data)

logging.info("-- 80 Generate info card")
Expand All @@ -101,109 +95,18 @@ def main():
search.add_ranking_combined(data)

logging.info("-- 99 Search: Export")
export_for_search(data, "output/search_data.json")

for _id, entry in data.items():
if entry["type"] != "root":
entry.setdefault("maps", {})["default"] = "interactive"
export.export_for_search(data, "output/search_data.json")

logging.info("-- 100 Export: API")
export_for_api(data, "output/api_data.json")

# Sitemap is only generated for deployment:
if DEBUG_MODE:
logging.info("Skipping sitemap generation in Dev Mode (GIT_COMMIT_SHA is unset)")
else:
logging.info("-- 101 Extra: Sitemap")
sitemap.generate_sitemap()


def export_for_search(data, path):
export = []
for _id, _data in data.items():
# Currently, the "root" entry is excluded from search
if _id == "root":
continue

if _data["type"] in {"room", "virtual_room"}:
building_parents_index = list(
map(lambda e: data[e]["type"] in {"building", "joined_building"},
_data["parents"])
).index(True)
else:
building_parents_index = len(_data["parents"])

export.append({
"ms_id": _id.replace(".", "-"), # MeiliSearch requires an id without "."; also this puts more emphasis on the order (because "." counts as more distance)
"id": _id, # not searchable
"name": _data["name"],
"arch_name": _data.get("tumonline_data", {}).get("arch_name", None),
"type": _data["type"],
"type_common_name": _data["type_common_name"],
"facet": {
"site": "site",
"campus": "site",
"area": "site",
"joined_building": "building",
"building": "building",
"room": "room",
"virtual_room": "room",
}.get(_data["type"], None),
# Parents always exclude root
###"parent_names": _data["parents"][1:],#[data[p]["name"] for p in _data["parents"][1:]],
# For rooms, the (joined_)building parents are extra to put more emphasis on them.
# Also their name is included
"parent_building": [data[p]["name"] for p in _data["parents"][building_parents_index:]],
# For all other parents, only the ids and their keywords (TODO) are searchable
"parent_keywords": _data["parents"][1:],
"address": _data.get("tumonline_data", {}).get("address", None),
"usage": _data.get("usage", {}).get("name", None),
"rank": int(_data["ranking_factors"]["rank_combined"]),
})

with open(path, "w") as f:
json.dump(export, f)


def export_for_api(data, path):
# Add some more information about parents
export_data = {}
for _id, _data in data.items():
export_data[_id] = {
"parent_names": [data[p]["name"] for p in _data["parents"]],
#"type_common_name": {
# "root": "Standortübersicht",
# "site": "Standort",
# "campus": "Campus",
# "area": "Gebiet / Gruppe von Gebäuden",
# "joined_building": "Gebäudekomplex",
# "building": "Gebäudeteil"
# if (_data["type"] == "building" and
# data[_data["parents"][-1]]["type"] == "joined_building")
# else "Gebäude",
# "room": _data["usage"]["name"] if "usage" in _data else "Raum",
# "virtual_room": _data["usage"]["name"] if "usage" in _data else "Raum/Gebäudeteil",
#}[_data["type"]],
**_data
}
if "children" in export_data[_id]:
del export_data[_id]["children"]
del export_data[_id]["children_flat"]
if "tumonline_data" in export_data[_id]:
del export_data[_id]["tumonline_data"]
if "roomfinder_data" in export_data[_id]:
del export_data[_id]["roomfinder_data"]
if "props" in export_data[_id]:
to_delete = list(filter(lambda e: e != "computed", export_data[_id]["props"].keys()))
for k in to_delete:
del export_data[_id]["props"][k]

with open(path, "w") as f:
json.dump(export_data, f)
export.export_for_api(data, "output/api_data.json")

# Sitemap is only generated for deployments
logging.info("-- 101 Extra: Sitemap")
sitemap.generate_sitemap()


if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG if DEBUG_MODE else logging.INFO, format='%(levelname)s: %(message)s')
logging.basicConfig(level=logging.DEBUG if DEBUG_MODE else logging.INFO, format="%(levelname)s: %(message)s")
logging.addLevelName(logging.INFO, f"\033[1;36m{logging.getLevelName(logging.INFO)}\033[1;0m")
logging.addLevelName(logging.WARNING, f"\033[1;33m{logging.getLevelName(logging.WARNING)}\033[1;0m")
logging.addLevelName(logging.ERROR, f"\033[1;41m{logging.getLevelName(logging.ERROR)}\033[1;0m")
Expand Down
Loading

0 comments on commit 19f9240

Please sign in to comment.