Copenhagen Phase 1: scaffolding + area + OSM download
- pyproject.toml with all deps (geopandas, contextily, partridge, etc.)
- config/: area.json (3 OSM boundary relations), modes.yaml (subway/s_tog/
light_rail/regional/bus, S-tog scoped to route=light_rail network=Takst),
styling.yaml (zorder/widths/colours/palette/mask/labels)
- build_area.py: fetch 3 relations via OSM API, stitch rings, dissolve to
area.gpkg/area.geojson (City Pass approximation)
- download_osm.py: per-mode Overpass (mirror+backoff, out geom + recurse for
stops), writes {mode}.gpkg (lines+stops) + query text
- download_gtfs.py / gtfs_to_geopackage.py: stubs (blocked on Rejseplanen)
- data/ gitignored (regenerable); output/ tracked
This commit is contained in:
marvin
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download transit route lines + stops from Overpass, per mode.
|
||||
|
||||
For each mode in config/modes.yaml, runs an Overpass query (with mirror
|
||||
fallback + exponential backoff) and writes:
|
||||
data/raw/osm/{mode}.gpkg (layers: "lines", "stops", EPSG:4326)
|
||||
data/raw/osm/{mode}.overpassql (the exact query text)
|
||||
|
||||
Lines: one MultiLineString per route relation (concatenation of member way
|
||||
geometries), carrying ref/name/network/colour/operator/route/mode tags.
|
||||
Stops: one Point per stop/platform node member, carrying ref/name/mode.
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import geopandas as gpd
|
||||
import requests
|
||||
import yaml
|
||||
from shapely.geometry import LineString, MultiLineString, Point
|
||||
|
||||
from _common import CONFIG, OSM_RAW, PROCESSED, UA
|
||||
|
||||
MIRRORS = [
|
||||
"https://overpass-api.de/api/interpreter",
|
||||
"https://overpass.kumi.systems/api/interpreter",
|
||||
"https://overpass.private.coffee/api/interpreter",
|
||||
]
|
||||
|
||||
PLATFORM_ROLES = {"platform", "platform_entry", "platform_exit"}
|
||||
STOP_ROLES = {"stop", "stop_entry", "stop_exit", "platform", "platform_entry", "platform_exit"}
|
||||
|
||||
|
||||
def build_query(mode_cfg, bbox):
|
||||
s, n, w, e = bbox
|
||||
filters = []
|
||||
for k, v in mode_cfg["osm"].items():
|
||||
if k.endswith("_neq"):
|
||||
filters.append(f'["{k[:-4]}"!="{v}"]')
|
||||
else:
|
||||
filters.append(f'["{k}"="{v}"]')
|
||||
filt = "".join(filters)
|
||||
return f"""[out:json][timeout:180];
|
||||
relation{filt}({s},{w},{n},{e});
|
||||
out geom;
|
||||
>;
|
||||
out body qt;
|
||||
"""
|
||||
|
||||
|
||||
def overpass(ql):
|
||||
last_err = None
|
||||
for mi, mirror in enumerate(MIRRORS):
|
||||
for attempt in range(4):
|
||||
try:
|
||||
r = requests.post(
|
||||
mirror,
|
||||
data={"data": ql},
|
||||
headers={"User-Agent": UA},
|
||||
timeout=300,
|
||||
)
|
||||
if r.status_code == 429 or r.status_code >= 500:
|
||||
raise RuntimeError(f"HTTP {r.status_code}")
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
wait = 2 ** (attempt + mi)
|
||||
print(f" [{mirror}] attempt {attempt+1} failed: {e}; retry in {wait}s", flush=True)
|
||||
time.sleep(wait)
|
||||
raise RuntimeError(f"all mirrors failed: {last_err}")
|
||||
|
||||
|
||||
def parse(data, mode):
|
||||
nodes = {}
|
||||
for e in data["elements"]:
|
||||
if e["type"] == "node":
|
||||
nodes[e["id"]] = e
|
||||
|
||||
lines, stops = [], []
|
||||
stop_seen = set()
|
||||
for e in data["elements"]:
|
||||
if e["type"] != "relation" or "tags" not in e:
|
||||
continue
|
||||
tags = e["tags"]
|
||||
if tags.get("route") is None:
|
||||
continue
|
||||
segs = []
|
||||
for m in e["members"]:
|
||||
if m["type"] == "way" and "geometry" in m and m.get("role") not in PLATFORM_ROLES:
|
||||
coords = [(g["lon"], g["lat"]) for g in m["geometry"]]
|
||||
if len(coords) >= 2:
|
||||
segs.append(LineString(coords))
|
||||
geom = MultiLineString(segs) if len(segs) > 1 else (segs[0] if segs else None)
|
||||
if geom is None:
|
||||
continue
|
||||
lines.append({
|
||||
"mode": mode,
|
||||
"ref": tags.get("ref"),
|
||||
"name": tags.get("name"),
|
||||
"network": tags.get("network"),
|
||||
"colour": tags.get("colour"),
|
||||
"operator": tags.get("operator"),
|
||||
"route": tags.get("route"),
|
||||
"geometry": geom,
|
||||
})
|
||||
for m in e["members"]:
|
||||
if m["type"] != "node" or m.get("role") not in STOP_ROLES:
|
||||
continue
|
||||
nid = m["ref"]
|
||||
if nid not in nodes or nid in stop_seen:
|
||||
continue
|
||||
nd = nodes[nid]
|
||||
if "lat" not in nd or "lon" not in nd:
|
||||
continue
|
||||
stop_seen.add(nid)
|
||||
ntags = nd.get("tags", {})
|
||||
stops.append({
|
||||
"mode": mode,
|
||||
"name": ntags.get("name") or ntags.get("public_transport") or "stop",
|
||||
"ref": ntags.get("ref"),
|
||||
"public_transport": ntags.get("public_transport"),
|
||||
"railway": ntags.get("railway"),
|
||||
"geometry": Point(nd["lon"], nd["lat"]),
|
||||
})
|
||||
return lines, stops
|
||||
|
||||
|
||||
def main():
|
||||
modes = yaml.safe_load((CONFIG / "modes.yaml").read_text())["modes"]
|
||||
area = gpd.read_file(PROCESSED / "area.gpkg")
|
||||
w, s, e, n = area.total_bounds # (minx, miny, maxx, maxy) = (west, south, east, north)
|
||||
|
||||
OSM_RAW.mkdir(parents=True, exist_ok=True)
|
||||
for mode, cfg in modes.items():
|
||||
ql = build_query(cfg, (s, n, w, e))
|
||||
(OSM_RAW / f"{mode}.overpassql").write_text(ql)
|
||||
print(f"[{mode}] querying Overpass bbox=({s:.4f},{w:.4f},{n:.4f},{e:.4f})...", flush=True)
|
||||
data = overpass(ql)
|
||||
lines, stops = parse(data, mode)
|
||||
print(f" -> {len(lines)} lines, {len(stops)} stops", flush=True)
|
||||
if not lines:
|
||||
continue
|
||||
lgdf = gpd.GeoDataFrame(lines, crs="EPSG:4326")
|
||||
lgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="lines")
|
||||
if stops:
|
||||
sgdf = gpd.GeoDataFrame(stops, crs="EPSG:4326")
|
||||
sgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="stops")
|
||||
print("done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user