#!/usr/bin/env python3
"""Builds Roomfeel's bundled places dataset from OpenStreetMap.

    python3 Tools/fetch_osm.py            # query Overpass and write the dataset
    python3 -m unittest Tools.test_fetch_osm   # run the parser tests (from the project root)

Data (c) OpenStreetMap contributors, available under the Open Database License (ODbL 1.0).
https://www.openstreetmap.org/copyright

Why a bundled extract instead of querying a server from the app: it works offline, it never sends where a person
is looking to anyone, the public Overpass servers ask apps not to hammer them, and the result is reproducible and
reviewable. Re-run this script to refresh it. Nothing here guesses: a tag that is missing stays missing, and opening
hours the parser cannot fully understand are left out (the app then shows no hours) rather than approximated.
"""
import json
import re
import sys
import time
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from math import cos, radians, sqrt
from pathlib import Path

OVERPASS = "https://overpass-api.de/api/interpreter"
USER_AGENT = "Roomfeel-pilot/0.1 (contact@appvantix.com)"
# Knoxville, Tennessee and the neighborhoods the pilot covers: south, west, north, east.
BBOX = (35.88, -84.10, 36.05, -83.80)
OUT = Path(__file__).resolve().parent.parent / "Roomfeel" / "Resources" / "OpenData" / "knoxville-places.json"

def query() -> str:
    b = "({},{},{},{})".format(*BBOX)
    return f"""[out:json][timeout:120];
(
  nwr["amenity"="cafe"]["name"]{b};
  nwr["amenity"~"^(restaurant|bar|pub|biergarten)$"]["name"]{b};
  nwr["shop"="books"]["name"]{b};
  nwr["amenity"="coworking_space"]["name"]{b};
  nwr["office"="coworking"]["name"]{b};
  nwr["leisure"~"^(escape_game|bowling_alley|amusement_arcade|miniature_golf|adult_gaming_centre)$"]["name"]{b};
  nwr["sport"="climbing"]["name"]{b};
  nwr["amenity"="arts_centre"]["name"]{b};
  nwr["shop"="games"]["name"]{b};
);
out center tags qt;
"""

# ---------------------------------------------------------------- categories

def category(tags: dict):
    amenity, shop, leisure = tags.get("amenity"), tags.get("shop"), tags.get("leisure")
    if amenity == "cafe": return "cafe"
    if amenity == "restaurant": return "restaurant"
    if amenity in ("bar", "pub", "biergarten"): return "bar"
    if shop == "books": return "bookstore"
    if amenity == "coworking_space" or tags.get("office") == "coworking": return "coworking"
    if leisure in ("escape_game", "bowling_alley", "amusement_arcade", "miniature_golf", "adult_gaming_centre"): return "activity"
    if tags.get("sport") == "climbing" or amenity == "arts_centre" or shop == "games": return "activity"
    return None

# ---------------------------------------------------------------- opening hours

DAY_NUM = {"Su": 1, "Mo": 2, "Tu": 3, "We": 4, "Th": 5, "Fr": 6, "Sa": 7}   # matches Calendar weekday numbers
DAY_ORDER = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"]
_DAY = r"(?:Mo|Tu|We|Th|Fr|Sa|Su)"
_DAY_ITEM = rf"{_DAY}(?:\s*-\s*{_DAY})?"
_DAYS = rf"{_DAY_ITEM}(?:\s*,\s*{_DAY_ITEM})*"
_TIME = r"\d{1,2}:\d{2}"
_SPAN = rf"{_TIME}\s*-\s*{_TIME}"
_SPANS = rf"{_SPAN}(?:\s*,\s*{_SPAN})*"
_RULE = re.compile(rf"^(?P<days>{_DAYS})?\s*(?P<rest>(?:{_SPANS})|off|closed)$", re.I)

def _days(selector):
    """'Mo-Fr', 'Sa,Su', 'Fr-Mo' -> set of Calendar weekday numbers (1=Sunday)."""
    if not selector:
        return set(DAY_NUM.values())
    out = set()
    for item in re.split(r"\s*,\s*", selector.strip()):
        parts = [p.strip().title() for p in item.split("-")]
        if len(parts) == 1:
            out.add(DAY_NUM[parts[0]])
        else:
            i, j = DAY_ORDER.index(parts[0]), DAY_ORDER.index(parts[1])
            k = i
            while True:
                out.add(DAY_NUM[DAY_ORDER[k]])
                if k == j: break
                k = (k + 1) % 7
    return out

def _minutes(text, end=False):
    """Minutes after midnight. An end time may run past 24:00 (OpenStreetMap writes 'until 3 a.m.' as 27:00)."""
    h, m = (int(x) for x in text.split(":"))
    limit = 48 if end else 24
    if h > limit or m > 59 or (h == limit and m > 0): raise ValueError(text)
    return h * 60 + m

def _spans(rest):
    out = []
    for a, b in re.findall(rf"({_TIME})\s*-\s*({_TIME})", rest):
        start, end = _minutes(a), _minutes(b, end=True)
        if end <= start: end += 1440          # open past midnight, e.g. 17:00-02:00
        out.append((start, end))
    return out

def split_rules(text):
    """Splits on ';', and on a comma that starts a new day selector after a time (an 'additional rule')."""
    rules = []
    for chunk in text.split(";"):
        pieces = re.split(rf"(?<=\d)\s*,\s*(?={_DAY})", chunk.strip())
        for n, piece in enumerate(pieces):
            if piece.strip():
                rules.append((piece.strip(), n > 0))   # (rule, is_additional)
    return rules

def parse_hours(text):
    """OpenStreetMap opening_hours -> [{"days": [..], "open": minutes, "close": minutes}], or None.

    None means "unknown": the string is empty, or uses anything this parser does not fully understand (public
    holidays, sunrise, week numbers, months, comments). Returning None is deliberate; a wrong time is worse than none.
    """
    if not text or not text.strip():
        return None
    text = text.strip()
    if text == "24/7":
        return [{"days": sorted(DAY_NUM.values()), "open": 0, "close": 1440}]
    per_day = {d: [] for d in DAY_NUM.values()}
    try:
        for rule, additional in split_rules(text):
            m = _RULE.match(rule)
            if not m:
                return None
            days = _days(m.group("days"))
            rest = m.group("rest").lower()
            if rest in ("off", "closed"):
                for d in days: per_day[d] = []
                continue
            spans = _spans(rest)
            for d in days:
                per_day[d] = (per_day[d] + spans) if additional else list(spans)
    except (ValueError, KeyError):
        return None
    # Group days that share exactly the same spans.
    groups = {}
    for d, spans in per_day.items():
        for span in sorted(set(spans)):
            groups.setdefault(span, []).append(d)
    out = [{"days": sorted(ds), "open": s, "close": e} for (s, e), ds in groups.items()]
    out.sort(key=lambda r: (r["open"], r["close"], r["days"]))
    return out or None   # a place that is "off" every day has no usable hours

# ---------------------------------------------------------------- facts

def facts(tags):
    out = {}
    wifi = tags.get("internet_access")
    if wifi in ("wlan", "yes", "wifi"):
        out["wifi"] = tags.get("internet_access:fee") != "yes"
        if tags.get("internet_access:fee") == "yes": out["wifi"] = False   # paid wifi is not offered as a plain "Wi-Fi"
    elif wifi == "no":
        out["wifi"] = False
    if tags.get("outdoor_seating") in ("yes", "no"): out["outdoorSeating"] = tags["outdoor_seating"] == "yes"
    if tags.get("wheelchair") in ("yes", "limited", "no"): out["wheelchair"] = tags["wheelchair"]
    if tags.get("reservation") in ("yes", "no", "required", "recommended"): out["reservations"] = tags["reservation"]
    if tags.get("takeaway") in ("yes", "no", "only"): out["takeaway"] = tags["takeaway"] != "no"
    return out

# ---------------------------------------------------------------- normalise

def website(tags):
    url = (tags.get("website") or tags.get("contact:website") or "").strip()
    if not url: return None
    if not re.match(r"^https?://", url, re.I): url = "https://" + url
    return url

def address(tags):
    street = tags.get("addr:street")
    if not street: return None
    number = tags.get("addr:housenumber")
    # A street with no house number is just the street. (An f-string of a missing number printed "None Main St".)
    return " ".join(part.strip() for part in (number, street) if part and part.strip())

def cuisine(tags):
    """Up to four cuisine codes ("pizza", "ice_cream"), in the order the mapper wrote them. Unknown shapes are dropped."""
    out = []
    for raw in (tags.get("cuisine") or "").split(";"):
        c = raw.strip().lower()
        if re.fullmatch(r"[a-z][a-z_]{1,30}", c) and c not in out:
            out.append(c)
    return out[:4]

def _host(url):
    m = re.match(r"^https?://([^/?#]+)", url, re.I)
    return m.group(1).lower() if m else None

def social(tags, kind):
    """The business's own Instagram or Facebook page as a clean https URL, or None.

    Mappers write a handle, an @handle, or a full URL. Only the two real hosts are accepted, and the query string
    is dropped, so a tag can never send someone anywhere else.
    """
    raw = (tags.get(f"contact:{kind}") or "").strip()
    if not raw:
        return None
    if kind == "instagram":
        if raw.lower().startswith("http"):
            if _host(raw) not in ("instagram.com", "www.instagram.com"): return None
            segment = re.sub(r"^https?://[^/]+/", "", raw, flags=re.I).split("?")[0].split("#")[0].strip("/").split("/")[0]
        else:
            segment = raw.lstrip("@")
        if not re.fullmatch(r"[A-Za-z0-9._]{1,30}", segment or "") or segment.lower() in ("p", "explore", "accounts", "reel"):
            return None
        return f"https://www.instagram.com/{segment}/"
    if kind == "facebook":
        if raw.lower().startswith("http"):
            if _host(raw) not in ("facebook.com", "www.facebook.com", "m.facebook.com"): return None
            path = re.sub(r"^https?://[^/]+/", "", raw, flags=re.I).split("?")[0].split("#")[0].strip("/")
        else:
            path = raw.lstrip("@")
        if not re.fullmatch(r"[A-Za-z0-9._\-/%]{3,100}", path or "") or path.lower().startswith(("sharer", "share", "login", "dialog")):
            return None
        return f"https://www.facebook.com/{path}/"
    return None

def metres(a, b):
    dy = (a[0] - b[0]) * 111_320
    dx = (a[1] - b[1]) * 111_320 * cos(radians((a[0] + b[0]) / 2))
    return sqrt(dx * dx + dy * dy)

def normalise(elements):
    places = []
    for e in elements:
        tags = e.get("tags", {})
        cat = category(tags)
        name = (tags.get("name") or "").strip()
        point = (e["lat"], e["lon"]) if "lat" in e else ((e["center"]["lat"], e["center"]["lon"]) if "center" in e else None)
        if not cat or not name or not point:
            continue
        kind = {"node": "n", "way": "w", "relation": "r"}[e["type"]]
        p = {
            "id": f"osm-{kind}{e['id']}",
            "name": name,
            "category": cat,
            "lat": round(point[0], 6),
            "lon": round(point[1], 6),
        }
        if address(tags): p["address"] = address(tags)
        phone = (tags.get("phone") or tags.get("contact:phone") or "").strip()
        if phone: p["phone"] = phone
        if website(tags): p["website"] = website(tags)
        raw_hours = (tags.get("opening_hours") or "").strip()
        if raw_hours:
            p["hoursText"] = raw_hours
            parsed = parse_hours(raw_hours)
            if parsed: p["hours"] = parsed
        f = facts(tags)
        if f: p["facts"] = f
        if cuisine(tags): p["cuisine"] = cuisine(tags)
        for kind in ("instagram", "facebook"):
            link = social(tags, kind)
            if link: p[kind] = link
        places.append(p)
    # A node and a way for the same business: keep the richer one.
    places.sort(key=lambda p: -len(p))
    kept = []
    for p in places:
        if any(q["name"].lower() == p["name"].lower() and metres((q["lat"], q["lon"]), (p["lat"], p["lon"])) < 60 for q in kept):
            continue
        kept.append(p)
    kept.sort(key=lambda p: (p["category"], p["name"].lower(), p["id"]))
    return kept

# ---------------------------------------------------------------- main

def fetch():
    body = urllib.parse.urlencode({"data": query()}).encode()
    req = urllib.request.Request(OVERPASS, data=body, headers={"User-Agent": USER_AGENT})
    last = None
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=180) as r:
                return json.load(r)["elements"]
        except Exception as error:   # Overpass is busy at times; a polite retry is fine
            last = error
            time.sleep(15 * (attempt + 1))
    raise SystemExit(f"Overpass failed: {last}")

def compare(old, new):
    """What a refresh changed, by place id: added, removed, and which of the shown fields differ."""
    before = {p["id"]: p for p in old}
    after = {p["id"]: p for p in new}
    fields = ("name", "address", "phone", "website", "hours", "facts", "cuisine", "instagram", "facebook")
    changed = {}
    for pid in before.keys() & after.keys():
        diff = [k for k in fields if before[pid].get(k) != after[pid].get(k)]
        if diff: changed[pid] = diff
    return {
        "added": sorted(after[i]["name"] for i in after.keys() - before.keys()),
        "removed": sorted(before[i]["name"] for i in before.keys() - after.keys()),
        "changed": changed,
    }

def main():
    force = "--force" in sys.argv
    old = []
    if OUT.exists():
        try: old = json.loads(OUT.read_text(encoding="utf-8")).get("places", [])
        except Exception: old = []
    elements = fetch()
    places = normalise(elements)
    # An Overpass server that answers with half a result would silently delete places, so a big drop needs --force.
    if old and len(places) < 0.8 * len(old) and not force:
        raise SystemExit(f"Refusing to write: {len(places)} places is under 80% of the current {len(old)}. "
                         "Overpass may have answered partially. Run again, or pass --force if the drop is real.")
    counts = {}
    for p in places: counts[p["category"]] = counts.get(p["category"], 0) + 1
    data = {
        "source": "OpenStreetMap contributors",
        "license": "Open Database License (ODbL) 1.0",
        "licenseURL": "https://www.openstreetmap.org/copyright",
        "extractedAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "area": {"name": "Knoxville, Tennessee", "south": BBOX[0], "west": BBOX[1], "north": BBOX[2], "east": BBOX[3]},
        "query": query(),
        "places": places,
    }
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(json.dumps(data, indent=1, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8")
    with_hours = sum(1 for p in places if "hours" in p)
    unparsed = sum(1 for p in places if "hoursText" in p and "hours" not in p)
    print(f"wrote {OUT} ({len(places)} places, {OUT.stat().st_size // 1024} KB)")
    print("by category:", counts)
    print(f"hours parsed for {with_hours}; {unparsed} hour strings left out as not fully understood")
    for key in ("address", "phone", "website", "facts", "cuisine", "instagram", "facebook"):
        print(f"  {key}: {sum(1 for p in places if key in p)}")
    if old:
        d = compare(old, places)
        print(f"since the last extract: {len(d['added'])} added, {len(d['removed'])} removed, {len(d['changed'])} changed")
        for label in ("added", "removed"):
            if d[label]: print(f"  {label}: " + ", ".join(d[label][:12]) + (" ..." if len(d[label]) > 12 else ""))

if __name__ == "__main__":
    main()