marvin
8e32ca2b1c
- prepare.py: merge per-mode OSM lines, resolve colours (GTFS route_color -> OSM colour tag -> keyed palette, robust to pd.NA), categorise buses (A/C/S/regular via regex), clip lines+stops to area, write master.gpkg layers (lines/stops/stations) - download_osm.py: add stations query (public_transport=station / railway=station nodes+areas, out center) -> stations.gpkg with proper human-readable names + subway/light_rail mode tags for labelling - verified: 135 lines (8 metro/16 s-tog/28 regional/83 bus), 90 named stations, zero NaN colours
216 lines
7.1 KiB
Python
216 lines
7.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Download transit route lines + stops from Overpass, per mode.
|
|
|
|
For each mode in config/modes.yaml, runs an Overpass query (with mirror
|
|
fallback + exponential backoff) and writes:
|
|
data/raw/osm/{mode}.gpkg (layers: "lines", "stops", EPSG:4326)
|
|
data/raw/osm/{mode}.overpassql (the exact query text)
|
|
|
|
Lines: one MultiLineString per route relation (concatenation of member way
|
|
geometries), carrying ref/name/network/colour/operator/route/mode tags.
|
|
Stops: one Point per stop/platform node member, carrying ref/name/mode.
|
|
"""
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import geopandas as gpd
|
|
import requests
|
|
import yaml
|
|
from shapely.geometry import LineString, MultiLineString, Point
|
|
|
|
from _common import CONFIG, OSM_RAW, PROCESSED, UA
|
|
|
|
MIRRORS = [
|
|
"https://overpass-api.de/api/interpreter",
|
|
"https://overpass.kumi.systems/api/interpreter",
|
|
"https://overpass.private.coffee/api/interpreter",
|
|
]
|
|
|
|
PLATFORM_ROLES = {"platform", "platform_entry", "platform_exit"}
|
|
STOP_ROLES = {"stop", "stop_entry", "stop_exit", "platform", "platform_entry", "platform_exit"}
|
|
|
|
|
|
def build_query(mode_cfg, bbox):
|
|
s, n, w, e = bbox
|
|
filters = []
|
|
for k, v in mode_cfg["osm"].items():
|
|
if k.endswith("_neq"):
|
|
filters.append(f'["{k[:-4]}"!="{v}"]')
|
|
else:
|
|
filters.append(f'["{k}"="{v}"]')
|
|
filt = "".join(filters)
|
|
return f"""[out:json][timeout:180];
|
|
relation{filt}({s},{w},{n},{e});
|
|
out geom;
|
|
>;
|
|
out body qt;
|
|
"""
|
|
|
|
|
|
def overpass(ql):
|
|
last_err = None
|
|
for mi, mirror in enumerate(MIRRORS):
|
|
for attempt in range(4):
|
|
try:
|
|
r = requests.post(
|
|
mirror,
|
|
data={"data": ql},
|
|
headers={"User-Agent": UA},
|
|
timeout=300,
|
|
)
|
|
if r.status_code == 429 or r.status_code >= 500:
|
|
raise RuntimeError(f"HTTP {r.status_code}")
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except Exception as e:
|
|
last_err = e
|
|
wait = 2 ** (attempt + mi)
|
|
print(f" [{mirror}] attempt {attempt+1} failed: {e}; retry in {wait}s", flush=True)
|
|
time.sleep(wait)
|
|
raise RuntimeError(f"all mirrors failed: {last_err}")
|
|
|
|
|
|
def parse(data, mode):
|
|
nodes = {}
|
|
for e in data["elements"]:
|
|
if e["type"] == "node":
|
|
nodes[e["id"]] = e
|
|
|
|
lines, stops = [], []
|
|
stop_seen = set()
|
|
for e in data["elements"]:
|
|
if e["type"] != "relation" or "tags" not in e:
|
|
continue
|
|
tags = e["tags"]
|
|
if tags.get("route") is None:
|
|
continue
|
|
segs = []
|
|
for m in e["members"]:
|
|
if m["type"] == "way" and "geometry" in m and m.get("role") not in PLATFORM_ROLES:
|
|
coords = [(g["lon"], g["lat"]) for g in m["geometry"]]
|
|
if len(coords) >= 2:
|
|
segs.append(LineString(coords))
|
|
geom = MultiLineString(segs) if len(segs) > 1 else (segs[0] if segs else None)
|
|
if geom is None:
|
|
continue
|
|
lines.append({
|
|
"mode": mode,
|
|
"ref": tags.get("ref"),
|
|
"name": tags.get("name"),
|
|
"network": tags.get("network"),
|
|
"colour": tags.get("colour"),
|
|
"operator": tags.get("operator"),
|
|
"route": tags.get("route"),
|
|
"geometry": geom,
|
|
})
|
|
for m in e["members"]:
|
|
if m["type"] != "node" or m.get("role") not in STOP_ROLES:
|
|
continue
|
|
nid = m["ref"]
|
|
if nid not in nodes or nid in stop_seen:
|
|
continue
|
|
nd = nodes[nid]
|
|
if "lat" not in nd or "lon" not in nd:
|
|
continue
|
|
stop_seen.add(nid)
|
|
ntags = nd.get("tags", {})
|
|
stops.append({
|
|
"mode": mode,
|
|
"name": ntags.get("name") or ntags.get("public_transport") or "stop",
|
|
"ref": ntags.get("ref"),
|
|
"public_transport": ntags.get("public_transport"),
|
|
"railway": ntags.get("railway"),
|
|
"geometry": Point(nd["lon"], nd["lat"]),
|
|
})
|
|
return lines, stops
|
|
|
|
|
|
def stations_query(bbox):
|
|
s, n, w, e = bbox
|
|
return f"""[out:json][timeout:180];
|
|
(
|
|
node["railway"="station"]({s},{w},{n},{e});
|
|
node["public_transport"="station"]({s},{w},{n},{e});
|
|
way["railway"="station"]({s},{w},{n},{e});
|
|
way["public_transport"="station"]({s},{w},{n},{e});
|
|
);
|
|
out center;
|
|
"""
|
|
|
|
|
|
def parse_stations(data):
|
|
"""Stations (nodes or area centroids) with proper human-readable names."""
|
|
rows = []
|
|
seen = set()
|
|
for e in data["elements"]:
|
|
if e["type"] not in ("node", "way"):
|
|
continue
|
|
tags = e.get("tags", {})
|
|
name = tags.get("name")
|
|
if not name:
|
|
continue
|
|
if e["type"] == "node":
|
|
geom = Point(e["lon"], e["lat"])
|
|
else:
|
|
c = e.get("center")
|
|
if not c:
|
|
continue
|
|
geom = Point(c["lon"], c["lat"])
|
|
key = (name, round(geom.x, 5), round(geom.y, 5))
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
rows.append({
|
|
"name": name,
|
|
"railway": tags.get("railway"),
|
|
"public_transport": tags.get("public_transport"),
|
|
"station": tags.get("station"),
|
|
"subway": tags.get("subway"),
|
|
"light_rail": tags.get("light_rail"),
|
|
"train": tags.get("train"),
|
|
"uic_ref": tags.get("uic_ref"),
|
|
"geometry": geom,
|
|
})
|
|
return rows
|
|
|
|
|
|
def main():
|
|
modes = yaml.safe_load((CONFIG / "modes.yaml").read_text())["modes"]
|
|
area = gpd.read_file(PROCESSED / "area.gpkg")
|
|
w, s, e, n = area.total_bounds # (minx, miny, maxx, maxy) = (west, south, east, north)
|
|
bbox = (s, n, w, e)
|
|
|
|
OSM_RAW.mkdir(parents=True, exist_ok=True)
|
|
for mode, cfg in modes.items():
|
|
ql = build_query(cfg, bbox)
|
|
(OSM_RAW / f"{mode}.overpassql").write_text(ql)
|
|
print(f"[{mode}] querying Overpass bbox=({s:.4f},{w:.4f},{n:.4f},{e:.4f})...", flush=True)
|
|
data = overpass(ql)
|
|
lines, stops = parse(data, mode)
|
|
print(f" -> {len(lines)} lines, {len(stops)} stops", flush=True)
|
|
if not lines:
|
|
continue
|
|
lgdf = gpd.GeoDataFrame(lines, crs="EPSG:4326")
|
|
lgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="lines")
|
|
if stops:
|
|
sgdf = gpd.GeoDataFrame(stops, crs="EPSG:4326")
|
|
sgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="stops")
|
|
|
|
# stations (named station nodes/areas) for labelling
|
|
sql = stations_query(bbox)
|
|
(OSM_RAW / "stations.overpassql").write_text(sql)
|
|
print("[stations] querying Overpass...", flush=True)
|
|
sdata = overpass(sql)
|
|
st = parse_stations(sdata)
|
|
print(f" -> {len(st)} named stations", flush=True)
|
|
if st:
|
|
gpd.GeoDataFrame(st, crs="EPSG:4326").to_file(
|
|
OSM_RAW / "stations.gpkg", driver="GPKG", layer="stations"
|
|
)
|
|
print("done.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|