jetlag-maps/copenhagen/scripts/download_osm.py

#!/usr/bin/env python3
"""Download transit route lines + stops from Overpass, per mode.

For each mode in config/modes.yaml, runs an Overpass query (with mirror
fallback + exponential backoff) and writes:
    data/raw/osm/{mode}.gpkg          (layers: "lines", "stops", EPSG:4326)
    data/raw/osm/{mode}.overpassql    (the exact query text)

Lines: one MultiLineString per route relation (concatenation of member way
geometries), carrying ref/name/network/colour/operator/route/mode tags.
Stops: one Point per stop/platform node member, carrying ref/name/mode.
"""
import sys
import time
from pathlib import Path

import geopandas as gpd
import requests
import yaml
from shapely.geometry import LineString, MultiLineString, Point

from _common import CONFIG, OSM_RAW, PROCESSED, UA

MIRRORS = [
    "https://overpass-api.de/api/interpreter",
    "https://overpass.kumi.systems/api/interpreter",
    "https://overpass.private.coffee/api/interpreter",
]

PLATFORM_ROLES = {"platform", "platform_entry", "platform_exit"}
STOP_ROLES = {"stop", "stop_entry", "stop_exit", "platform", "platform_entry", "platform_exit"}


def build_query(mode_cfg, bbox):
    s, n, w, e = bbox
    filters = []
    for k, v in mode_cfg["osm"].items():
        if k.endswith("_neq"):
            filters.append(f'["{k[:-4]}"!="{v}"]')
        else:
            filters.append(f'["{k}"="{v}"]')
    filt = "".join(filters)
    return f"""[out:json][timeout:180];
relation{filt}({s},{w},{n},{e});
out geom;
>;
out body qt;
"""


def overpass(ql):
    last_err = None
    for mi, mirror in enumerate(MIRRORS):
        for attempt in range(4):
            try:
                r = requests.post(
                    mirror,
                    data={"data": ql},
                    headers={"User-Agent": UA},
                    timeout=300,
                )
                if r.status_code == 429 or r.status_code >= 500:
                    raise RuntimeError(f"HTTP {r.status_code}")
                r.raise_for_status()
                return r.json()
            except Exception as e:
                last_err = e
                wait = 2 ** (attempt + mi)
                print(f"  [{mirror}] attempt {attempt+1} failed: {e}; retry in {wait}s", flush=True)
                time.sleep(wait)
    raise RuntimeError(f"all mirrors failed: {last_err}")


def parse(data, mode):
    nodes = {}
    for e in data["elements"]:
        if e["type"] == "node":
            nodes[e["id"]] = e

    lines, stops = [], []
    stop_seen = set()
    for e in data["elements"]:
        if e["type"] != "relation" or "tags" not in e:
            continue
        tags = e["tags"]
        if tags.get("route") is None:
            continue
        segs = []
        for m in e["members"]:
            if m["type"] == "way" and "geometry" in m and m.get("role") not in PLATFORM_ROLES:
                coords = [(g["lon"], g["lat"]) for g in m["geometry"]]
                if len(coords) >= 2:
                    segs.append(LineString(coords))
        geom = MultiLineString(segs) if len(segs) > 1 else (segs[0] if segs else None)
        if geom is None:
            continue
        lines.append({
            "mode": mode,
            "ref": tags.get("ref"),
            "name": tags.get("name"),
            "network": tags.get("network"),
            "colour": tags.get("colour"),
            "operator": tags.get("operator"),
            "route": tags.get("route"),
            "geometry": geom,
        })
        for m in e["members"]:
            if m["type"] != "node" or m.get("role") not in STOP_ROLES:
                continue
            nid = m["ref"]
            if nid not in nodes or nid in stop_seen:
                continue
            nd = nodes[nid]
            if "lat" not in nd or "lon" not in nd:
                continue
            stop_seen.add(nid)
            ntags = nd.get("tags", {})
            stops.append({
                "mode": mode,
                "name": ntags.get("name") or ntags.get("public_transport") or "stop",
                "ref": ntags.get("ref"),
                "public_transport": ntags.get("public_transport"),
                "railway": ntags.get("railway"),
                "geometry": Point(nd["lon"], nd["lat"]),
            })
    return lines, stops


def main():
    modes = yaml.safe_load((CONFIG / "modes.yaml").read_text())["modes"]
    area = gpd.read_file(PROCESSED / "area.gpkg")
    w, s, e, n = area.total_bounds  # (minx, miny, maxx, maxy) = (west, south, east, north)

    OSM_RAW.mkdir(parents=True, exist_ok=True)
    for mode, cfg in modes.items():
        ql = build_query(cfg, (s, n, w, e))
        (OSM_RAW / f"{mode}.overpassql").write_text(ql)
        print(f"[{mode}] querying Overpass bbox=({s:.4f},{w:.4f},{n:.4f},{e:.4f})...", flush=True)
        data = overpass(ql)
        lines, stops = parse(data, mode)
        print(f"  -> {len(lines)} lines, {len(stops)} stops", flush=True)
        if not lines:
            continue
        lgdf = gpd.GeoDataFrame(lines, crs="EPSG:4326")
        lgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="lines")
        if stops:
            sgdf = gpd.GeoDataFrame(stops, crs="EPSG:4326")
            sgdf.to_file(OSM_RAW / f"{mode}.gpkg", driver="GPKG", layer="stops")
    print("done.")


if __name__ == "__main__":
    main()