hulthe/jetlag-maps
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Source Copenhagen transit data from Rejseplanen GTFS

Browse Source 
- Replace OSM Overpass transit data with the Rejseplanen GTFS feed
  (routes keyed by (agency, short name); styles from modes.yaml)
- Draw one shape per (style, ref, direction), choosing the shape that
  serves the most in-area stops so lines pass the stops we show
- Collapse stops by names (verified unambiguous); prune stops whose
  serving refs have no drawn line within 300 m
- Patch Københavns Havn + Nordhavn into the area polygon so ferry
  routes and sub-harbour metro tunnels survive clipping; 100 m buffer
  closes relation boundary slivers
- Drop OSM download pipeline; keep tiled basemap
This commit is contained in:
marvin
2026-09-17 21:07:40 +02:00
parent 2dcaed7600
commit a6715eaff5
12 changed files with 522 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
copenhagen/scripts/prepare.py
+261 -146
View File
@@ -1,25 +1,44 @@
#!/usr/bin/env python3
"""Merge OSM layers into a single master GeoPackage, enriched for rendering.
"""Merge GTFS-derived layers into a single master GeoPackage, enriched for rendering.
Reads data/raw/osm/{mode}.gpkg (lines + stops) and data/processed/area.gpkg,
and writes data/processed/master.gpkg with layers "lines" and "stops".
Reads (from data/processed):
gtfs_shapes.gpkg route-tagged line geometries (gtfs_to_geopackage.py)
gtfs_stops.gpkg raw stop poles (one row per physical pole)
route_colors.csv route metadata (agency_id, short/long name, type, colour)
area.gpkg City Pass area polygon (build_area.py, incl. waterways)
and from data/raw/gtfs: agency.txt, trips.txt, stop_times.txt.
Enrichment:
- style : from config/modes.yaml (mode -> style key)
- colour : GTFS route_color (if present) -> OSM colour tag -> palette
- bus_category : A / C / S / regular (regex from styling.yaml)
Lines and stops are clipped to the City Pass area polygon.
Writes data/processed/master.gpkg with layers:
lines one feature per (style, ref, direction) — the shape that serves
the most in-area stops (so drawn lines pass the drawn stops)
stops bus/ferry stops, one point per stop name, pruned to stops
actually served by a drawn line (see STOP_LINE_MARGIN_M)
stations rail-family stations, one point per (name, style)
Stop names in the Rejseplanen feed are unambiguous per location (verified:
max spread within a name is ~350 m), so every merge is key-based on names —
no distance-based clustering anywhere.
Style classification comes from config/modes.yaml (agency + route_type);
file order is the priority when a pole is served by several modes.
Colours: GTFS route_color -> styling.yaml palette (the feed leaves
route_color empty for the Copenhagen operators, so the palette wins).
"""
import re
from pathlib import Path
import sys
import geopandas as gpd
import pandas as pd
import yaml
from _common import CONFIG, OSM_RAW, PROCESSED
from _common import CONFIG, GTFS_RAW, PROCESSED
MIXED_CRS_WARN = "mixed CRS"
LENGTH_CRS = "EPSG:25832" # UTM 32N — metres, for length/centroid computations
STATION_STYLES = {"metro", "s_tog", "light_rail", "regional"}
# A stop is drawn only if a drawn line of one of its serving refs passes
# within this distance. Routes have variants; without this check a stop can
# end up further from the map than the variant we chose not to draw.
STOP_LINE_MARGIN_M = 300
def load_modes():
@@ -30,173 +49,269 @@ def load_styling():
return yaml.safe_load((CONFIG / "styling.yaml").read_text())
def collect_lines(modes):
frames = []
for mode, cfg in modes.items():
p = OSM_RAW / f"{mode}.gpkg"
if not p.exists():
continue
g = gpd.read_file(p, layer="lines")
if g.empty:
continue
g["mode"] = mode
g["style"] = cfg["style"]
g["gtfs_route_type"] = cfg.get("gtfs_route_type")
frames.append(g)
if not frames:
return gpd.GeoDataFrame(columns=["mode", "ref", "style", "geometry"], crs="EPSG:4326")
return pd.concat(frames, ignore_index=True)
def load_area():
area = gpd.read_file(PROCESSED / "area.gpkg")
return area.geometry.union_all()
def collect_stops(modes):
frames = []
for mode, cfg in modes.items():
p = OSM_RAW / f"{mode}.gpkg"
if not p.exists():
def classify(agency, route_type, modes):
"""Map a GTFS (agency_name, route_type) pair to a style key, or None."""
for style, cfg in modes.items():
if agency not in cfg.get("agencies", []):
continue
try:
g = gpd.read_file(p, layer="stops")
except Exception:
types = cfg.get("route_types")
if types and route_type not in types:
continue
if g.empty:
continue
g["mode"] = mode
g["style"] = cfg["style"]
frames.append(g)
if not frames:
return None
return pd.concat(frames, ignore_index=True)
return style
return None
def gtfs_colour_map():
"""Return {(route_short_name, route_type): route_color} if GTFS data exists."""
csv = PROCESSED / "route_colors.csv"
if not csv.exists():
return {}
df = pd.read_csv(csv)
out = {}
for _, r in df.iterrows():
name = str(r.get("route_short_name") or "").strip()
if name:
out[(name, r.get("route_type"))] = r.get("route_color")
return out
def load_routes(modes):
"""route_colors.csv joined with agency names and classified by style."""
agencies = pd.read_csv(GTFS_RAW / "agency.txt", dtype=str)
agency_names = dict(zip(agencies["agency_id"], agencies["agency_name"]))
routes = pd.read_csv(PROCESSED / "route_colors.csv", dtype=str)
routes["route_type"] = pd.to_numeric(routes["route_type"], errors="coerce")
routes["agency_name"] = routes["agency_id"].map(agency_names)
routes["style"] = [
classify(a, t, modes)
for a, t in zip(routes["agency_name"], routes["route_type"])
]
return routes
def categorise_bus(ref, patterns):
if ref is None:
return "regular"
s = str(ref)
s = "" if ref is None else str(ref)
for cat, pat in patterns.items():
if re.search(pat, s):
return cat
return "regular"
def resolve_colour(row, palette, gtfs):
style = row["style"]
ref = row.get("ref")
colour = row.get("colour")
# 1. GTFS (if available)
if gtfs and isinstance(ref, str):
c = gtfs.get((ref, row.get("gtfs_route_type")))
if c:
return str(c)
# 2. OSM colour tag (only accept real hex/named strings, not NA/None)
if isinstance(colour, str) and colour.strip():
return colour
# 3. palette
def resolve_colour(style, ref, bus_category, gtfs_colour, palette):
# 1. GTFS route_color (mostly empty in this feed)
if isinstance(gtfs_colour, str) and gtfs_colour.strip():
return gtfs_colour
# 2. palette
p = palette.get(style, {})
if style == "bus":
cat = row.get("bus_category")
if not isinstance(cat, str):
cat = "regular"
return p.get(cat) or p.get("default")
key = ref if isinstance(ref, str) else None
return p.get(key) or p.get("default")
return p.get(bus_category) or p.get("default")
return p.get(ref) or p.get("default")
def build_lines(routes, styling, area_geom, trips, st, area_stop_ids):
"""One feature per (style, ref, direction).
Geometry per group: the shape that serves the most in-area stops
(tie-break: longest). Picking by length alone can draw a variant that
skips stops shown on the map (terminal stubs, short-turn branches).
"""
shapes = gpd.read_file(PROCESSED / "gtfs_shapes.gpkg")
shape_rows = trips.dropna(subset=["shape_id"]).drop_duplicates("shape_id")
shape_direction = dict(zip(shape_rows["shape_id"], shape_rows["direction_id"]))
# coverage: distinct in-area stops visited per shape
trip_shape = dict(zip(trips["trip_id"], trips["shape_id"]))
sv = st.assign(shape_id=st["trip_id"].map(trip_shape))
sv = sv[sv["stop_id"].isin(area_stop_ids)]
coverage = sv.groupby("shape_id")["stop_id"].nunique()
use = routes.dropna(subset=["style"])[
["route_id", "route_short_name", "route_long_name",
"route_color", "style"]
]
g = shapes.merge(use, on="route_id", how="inner")
g["direction_id"] = g["shape_id"].map(shape_direction)
g = g.to_crs(LENGTH_CRS)
g["_len"] = g.geometry.length
g["_cov"] = g["shape_id"].map(coverage).fillna(0)
g = (
g.sort_values(["_cov", "_len"], ascending=False)
.drop_duplicates(["style", "route_short_name", "direction_id"])
.to_crs("EPSG:4326")
)
g["ref"] = g["route_short_name"]
g["name"] = [
ln if isinstance(ln, str) and ln.strip() else ref
for ln, ref in zip(g["route_long_name"], g["ref"])
]
patterns = styling["bus_filters"]["categories"]
g["bus_category"] = [
categorise_bus(ref, patterns) if style == "bus" else None
for ref, style in zip(g["ref"], g["style"])
]
palette = styling["palette"]
g["colour_final"] = [
resolve_colour(style, ref, cat, gtfs_c, palette)
for style, ref, cat, gtfs_c
in zip(g["style"], g["ref"], g["bus_category"], g["route_color"])
]
g = g[["ref", "name", "style", "bus_category", "colour_final", "geometry"]]
g = g[~g.geometry.isna() & g.geometry.is_valid]
# the area polygon includes patched-in waterways (area.json), so ferry
# shapes and sub-harbour metro tunnels survive the clip unfragmented
return gpd.clip(g, area_geom)
def build_pole_classes(routes, modes, trips, st, area_stop_ids):
"""Per-pole classification and serving refs, from stop_times.
Returns (pole_styles, pole_refs):
pole_styles[stop_id] = highest-priority style among classified serving
routes (modes.yaml order)
pole_refs[stop_id] = set of serving route short names (classified)
"""
classified = routes.dropna(subset=["style"])
route_style = dict(zip(classified["route_id"], classified["style"]))
route_ref = dict(
zip(classified["route_id"], classified["route_short_name"])
)
trip_route = dict(zip(trips["trip_id"], trips["route_id"]))
sv = st[st["stop_id"].isin(area_stop_ids)].copy()
sv["route_id"] = sv["trip_id"].map(trip_route)
sv = sv.dropna(subset=["route_id"])
sv["style"] = sv["route_id"].map(route_style)
sv = sv.dropna(subset=["style"])
order = {s: i for i, s in enumerate(modes)}
sv["_p"] = sv["style"].map(order)
best = sv.loc[sv.groupby("stop_id")["_p"].idxmin(), ["stop_id", "style"]]
pole_styles = dict(zip(best["stop_id"], best["style"]))
pole_refs = sv.groupby("stop_id")["route_id"].apply(
lambda ids: {route_ref[i] for i in ids}
).to_dict()
return pole_styles, pole_refs
def build_stops_and_stations(pole_styles, pole_refs, area_geom, lines):
poles = gpd.read_file(PROCESSED / "gtfs_stops.gpkg")
poles["style"] = poles["stop_id"].map(pole_styles)
poles["refs"] = poles["stop_id"].map(lambda s: pole_refs.get(s, set()))
poles = poles.dropna(subset=["style"])
# project for accurate centroids and distances
poles = poles.to_crs(LENGTH_CRS)
# stops (everything not rail-family: buses + harbour ferries), one row
# per name, union of serving refs across its poles
not_rail = poles[~poles["style"].isin(STATION_STYLES)]
stop_rows = []
for (name, style), grp in not_rail.groupby(["stop_name", "style"]):
refs = set().union(*grp["refs"]) if len(grp) else set()
stop_rows.append({
"name": name,
"style": style,
"n_poles": len(grp),
"refs": refs,
"geometry": grp.geometry.union_all().centroid,
})
# stations: one row per (name, style)
rail = poles[poles["style"].isin(STATION_STYLES)]
station_rows = []
for (name, style), grp in rail.groupby(["stop_name", "style"]):
station_rows.append({
"name": name,
"style": style,
"n_poles": len(grp),
"geometry": grp.geometry.union_all().centroid,
})
area_25832 = gpd.GeoSeries([area_geom], crs="EPSG:4326").to_crs(LENGTH_CRS).union_all()
stops = gpd.GeoDataFrame(stop_rows, crs=LENGTH_CRS)
# prune stops not served by any drawn line within STOP_LINE_MARGIN_M
if len(stops) and len(lines):
line_geom = (
lines.to_crs(LENGTH_CRS)
.groupby("ref")["geometry"]
.agg(lambda g: g.union_all())
.to_dict()
)
keep = []
for _, r in stops.iterrows():
dists = [
geom.distance(r.geometry)
for rf in r["refs"]
if (geom := line_geom.get(rf)) is not None
]
keep.append(bool(dists) and min(dists) <= STOP_LINE_MARGIN_M)
n_dropped = (~pd.Series(keep, index=stops.index)).sum()
if n_dropped:
print(f"pruned {n_dropped} stops not served by a drawn line "
f"(>{STOP_LINE_MARGIN_M} m from nearest)", flush=True)
stops = stops[keep].drop(columns=["refs"])
else:
stops = stops.drop(columns=["refs"])
stops = gpd.clip(stops, area_25832)
stations = gpd.clip(
gpd.GeoDataFrame(station_rows, crs=LENGTH_CRS), area_25832
)
return stops.to_crs("EPSG:4326"), stations.to_crs("EPSG:4326")
def main():
for dep in ("gtfs_shapes.gpkg", "gtfs_stops.gpkg", "route_colors.csv"):
if not (PROCESSED / dep).exists():
print(f"prepare: missing {PROCESSED / dep} — "
"run gtfs_to_geopackage.py first.", file=sys.stderr)
sys.exit(2)
modes = load_modes()
styling = load_styling()
palette = styling["palette"]
bus_patterns = styling["bus_filters"]["categories"]
area_geom = load_area()
area = gpd.read_file(PROCESSED / "area.gpkg")
area_geom = area.geometry.union_all()
routes = load_routes(modes)
print(f"routes classified: {routes['style'].notna().sum()} of "
f"{len(routes)} map to a style", flush=True)
lines = collect_lines(modes)
print(f"collected {len(lines)} raw lines across modes", flush=True)
area_stop_ids = set(
gpd.read_file(PROCESSED / "gtfs_stops.gpkg")["stop_id"]
)
trips = pd.read_csv(
GTFS_RAW / "trips.txt",
usecols=["trip_id", "route_id", "shape_id", "direction_id"],
dtype=str,
)
st = pd.read_csv(
GTFS_RAW / "stop_times.txt",
usecols=["trip_id", "stop_id"],
dtype=str,
)
# bus categorisation (only meaningful for bus style)
lines["bus_category"] = [
categorise_bus(r, bus_patterns) if s == "bus" else None
for r, s in zip(lines.get("ref"), lines["style"])
]
lines = build_lines(routes, styling, area_geom, trips, st, area_stop_ids)
print(f"lines: {len(lines)} features (one per style/ref/direction)",
flush=True)
gtfs = gtfs_colour_map()
if gtfs:
print(f"GTFS colours loaded: {len(gtfs)} routes", flush=True)
else:
print("no GTFS colour data; using OSM colour -> palette", flush=True)
pole_styles, pole_refs = build_pole_classes(
routes, modes, trips, st, area_stop_ids
)
stops, stations = build_stops_and_stations(
pole_styles, pole_refs, area_geom, lines
)
print(f"stops: {len(stops)} (bus + ferry); stations: {len(stations)}",
flush=True)
lines["colour_final"] = [
resolve_colour(r, palette, gtfs) for _, r in lines.iterrows()
]
# clip to area
lines = lines[~lines.geometry.isna() & lines.geometry.is_valid]
clipped = gpd.clip(lines, area_geom)
print(f"clipped to area: {len(clipped)} line features remain", flush=True)
out_cols = [
"mode", "style", "ref", "name", "network", "operator", "route",
"colour", "bus_category", "colour_final", "geometry",
]
for c in out_cols:
if c not in clipped.columns:
clipped[c] = None
clipped = clipped.set_geometry("geometry")
clipped = gpd.GeoDataFrame(clipped[out_cols], crs="EPSG:4326")
PROCESSED.mkdir(parents=True, exist_ok=True)
# drop existing master.gpkg so layer overwrite is clean
out = PROCESSED / "master.gpkg"
if out.exists():
out.unlink()
clipped.to_file(out, driver="GPKG", layer="lines")
print(f"wrote lines layer: {len(clipped)} features", flush=True)
lines.to_file(out, driver="GPKG", layer="lines")
print(f"wrote lines layer: {len(lines)} features", flush=True)
stops.to_file(out, driver="GPKG", layer="stops")
print(f"wrote stops layer: {len(stops)} features", flush=True)
stations.to_file(out, driver="GPKG", layer="stations")
print(f"wrote stations layer: {len(stations)} features", flush=True)
# stops (stop_positions along routes)
stops = collect_stops(modes)
if stops is not None and not stops.empty:
stops = stops[~stops.geometry.isna() & stops.geometry.is_valid]
stops_clipped = gpd.clip(stops, area_geom)
keep = ["mode", "style", "name", "ref", "public_transport", "railway", "route_refs", "geometry"]
for c in keep:
if c not in stops_clipped.columns:
stops_clipped[c] = None
stops_clipped = gpd.GeoDataFrame(stops_clipped[keep], crs="EPSG:4326")
stops_clipped.to_file(out, driver="GPKG", layer="stops")
print(f"wrote stops layer: {len(stops_clipped)} features", flush=True)
else:
print("no stops to write", flush=True)
# stations (named station nodes/areas) for labelling
st_path = OSM_RAW / "stations.gpkg"
if st_path.exists():
st = gpd.read_file(st_path, layer="stations")
if not st.empty:
st = st[~st.geometry.isna() & st.geometry.is_valid]
st_clipped = gpd.clip(st, area_geom)
st_clipped = gpd.GeoDataFrame(st_clipped, crs="EPSG:4326")
st_clipped.to_file(out, driver="GPKG", layer="stations")
print(f"wrote stations layer: {len(st_clipped)} named stations", flush=True)
else:
print("no stations file; labelling will be limited", flush=True)
# summary by style
print("\nsummary by style:")
for style, grp in clipped.groupby("style"):
for style, grp in lines.groupby("style"):
refs = grp["ref"].dropna().unique()
print(f" {style:10s}: {len(grp):4d} feats, {len(refs):3d} refs")