#!/usr/bin/env python3
"""Rebuild the pinned, offline world geography snapshot (Python standard library only)."""

import csv
import gzip
import hashlib
import io
import json
import urllib.request
from pathlib import Path


RELEASE = "v5.1.0-export.1"
COMMIT = "925204e26d6a478f6b8af56eb516030b00e38e0a"
BASE = f"https://github.com/dr5hn/countries-states-cities-database/releases/download/{RELEASE}"
SOURCES = {
    "json-countries%2Bstates%2Bcities.json.gz": "6d1d618ace347a70c90f9bbfed5648adce444db1acf0899749fbbe71bff4beb8",
    "csv-translations.csv.gz": "f7941643bfaf2fe7f912af4457b8e3f3662793519656762833fd9799d65fc52e",
}
DEST = Path(__file__).resolve().parents[1] / "database/data/geography"


def download(name):
    with urllib.request.urlopen(f"{BASE}/{name}") as response:
        data = response.read()
    if hashlib.sha256(data).hexdigest() != SOURCES[name]:
        raise ValueError(f"Source checksum mismatch: {name}")
    return data


def main():
    DEST.mkdir(parents=True, exist_ok=True)
    countries = json.loads(gzip.decompress(download(next(iter(SOURCES)))))
    translations = {}
    with gzip.GzipFile(fileobj=io.BytesIO(download("csv-translations.csv.gz"))) as source:
        for row in csv.DictReader(io.TextIOWrapper(source, encoding="utf-8")):
            if row["language"] == "ar":
                translations[(row["place_type"], int(row["place_id"]))] = row["translation"]

    def names(kind, row):
        return {"en": row["name"], "ar": translations.get((kind, row["id"]), row["name"])}

    def area(row):
        return {"code": f"csc-city-{row['id']}", "name": names("city", row)}

    counts = {"countries": 0, "cities": 0, "areas": 0}
    payload = bytearray()
    for country in sorted(countries, key=lambda row: row["iso2"]):
        states = [
            {
                "code": state.get("iso3166_2") or f"csc-state-{state['id']}",
                "name": names("state", state),
                "areas": [area(city) for city in state["cities"]],
            }
            for state in country["states"]
        ]
        # Some source cities have no state. Keep them under a clearly named
        # container rather than dropping them or inventing an administrative unit.
        if country.get("cities"):
            states.append({
                "code": f"{country['iso2']}-unassigned",
                "name": {"en": "Other areas", "ar": "مناطق أخرى"},
                "areas": [area(city) for city in country["cities"]],
            })
        record = {
            "code": country["iso2"],
            "name": names("country", country),
            "phone_code": "+" + country["phonecode"].lstrip("+"),
            "currency_code": country.get("currency") or None,
            "cities": states,
        }
        counts["countries"] += 1
        counts["cities"] += len(states)
        counts["areas"] += sum(len(state["areas"]) for state in states)
        payload.extend((json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n").encode())

    snapshot = gzip.compress(bytes(payload), mtime=0)
    (DEST / "world.jsonl.gz").write_bytes(snapshot)
    manifest = {
        "source": "https://github.com/dr5hn/countries-states-cities-database",
        "release": RELEASE,
        "commit": COMMIT,
        "license": "ODbL-1.0",
        "source_sha256": SOURCES,
        "sha256": hashlib.sha256(snapshot).hexdigest(),
        "counts_before_kuwait_override": counts,
    }
    (DEST / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    # Keep every upstream distribution notice with the derived dataset.
    for name in ["NOTICE-4-72-Clausula-Licencia-Abierta.txt", "NOTICE-pc_rebuild_7-distribution-notices.txt",
                 "NOTICE-Posti-postal-code-terms.txt", "NOTICE-SOURCES.txt", "NOTICE-Unicode-3.0.txt"]:
        with urllib.request.urlopen(f"{BASE}/{name}") as response:
            (DEST / name).write_bytes(response.read())
    with urllib.request.urlopen(f"https://raw.githubusercontent.com/dr5hn/countries-states-cities-database/{COMMIT}/LICENSE") as response:
        (DEST / "LICENSE-ODbL.txt").write_bytes(response.read())
    print(json.dumps(manifest, indent=2))


if __name__ == "__main__":
    main()