"""Scoutline — multi-source Fetcher + Signalizer.

Each source pulls a public defensive feed and emits normalized Case objects
carrying observables + source metadata only; classification, mapping, sealing,
and routing happen downstream. Sources are registered in SOURCES; the
`source_metadata["feed"]` tag lets Classifyline assign the right incident type.
"""

from __future__ import annotations

import csv
import io
import os
from datetime import datetime, timedelta, timezone
from typing import Any, Callable, Dict, Iterable, List, Optional
from urllib.parse import urlparse

import httpx

from psyc import log
from psyc.models import Case, IncidentType, Observables


# CISA's CDN 403s "exotic" UAs from some IPs; a Mozilla-compatible identifier
# is universally accepted and still identifies us honestly. Overridable via env
# if a feed ever wants a specific UA.
USER_AGENT = os.environ.get(
    "PSYC_HTTP_USER_AGENT",
    "Mozilla/5.0 (compatible; psyc/0.1; +https://psyc.neuronetz.ai)",
)
HTTP_TIMEOUT = 30.0

URLHAUS_RECENT_CSV = "https://urlhaus.abuse.ch/downloads/csv_recent/"
CISA_KEV_JSON = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
FEODO_BLOCKLIST_JSON = "https://feodotracker.abuse.ch/downloads/ipblocklist.json"
THREATFOX_API = "https://threatfox-api.abuse.ch/api/v1/"
MALWARE_BAZAAR_API = "https://mb-api.abuse.ch/api/v1/"
OTX_PULSES_API = "https://otx.alienvault.com/api/v1/pulses/subscribed"

_log = log.get(__name__)


def _http(
    method: str,
    url: str,
    headers: Optional[Dict[str, str]] = None,
    json_body: Optional[Dict[str, Any]] = None,
    form_body: Optional[Dict[str, Any]] = None,
    params: Optional[Dict[str, Any]] = None,
    timeout: float = HTTP_TIMEOUT,
) -> httpx.Response:
    h = {"User-Agent": USER_AGENT}
    if headers:
        h.update(headers)
    with httpx.Client(timeout=timeout, headers=h, follow_redirects=True) as client:
        if method.upper() == "POST":
            if form_body is not None:
                resp = client.post(url, data=form_body, params=params)
            else:
                resp = client.post(url, json=json_body, params=params)
        else:
            resp = client.get(url, params=params)
        resp.raise_for_status()
        return resp


def _http_get(url: str) -> httpx.Response:
    return _http("GET", url)


def _parse_dt(value: str, fmt: str) -> datetime:
    try:
        return datetime.strptime(value, fmt).replace(tzinfo=timezone.utc)
    except (ValueError, TypeError):
        return datetime.now(timezone.utc)


# --- URLhaus — malware distribution URLs ---------------------------------

def _parse_urlhaus_csv(csv_text: str) -> Iterable[Dict[str, str]]:
    lines = [ln for ln in csv_text.splitlines() if ln and not ln.startswith("#")]
    if not lines:
        return
    reader = csv.reader(io.StringIO("\n".join(lines)), quotechar='"')
    for fields in reader:
        if len(fields) < 9:
            continue
        yield {
            "id": fields[0], "dateadded": fields[1], "url": fields[2],
            "url_status": fields[3], "last_online": fields[4], "threat": fields[5],
            "tags": fields[6], "urlhaus_link": fields[7], "reporter": fields[8],
        }


def _urlhaus_row_to_case(row: Dict[str, str]) -> Case:
    url = row["url"]
    host = urlparse(url).hostname or ""
    tags = [t.strip() for t in row["tags"].split(",") if t.strip()]
    summary = f"URLhaus: {row['threat'] or 'malware_distribution'} at {host or url}"
    if tags:
        summary += f" (tags: {', '.join(tags[:4])})"
    return Case(
        case_id=f"PSYC-URLHAUS-{row['id']}",
        summary=summary,
        source_type="abuse_feed",
        source_ref=row["urlhaus_link"],
        source_metadata=dict(
            feed="urlhaus", url_status=row["url_status"], threat=row["threat"],
            tags=row["tags"], reporter=row["reporter"],
        ),
        observed_at=_parse_dt(row["dateadded"], "%Y-%m-%d %H:%M:%S"),
        observables=Observables(urls=[url], domains=[host] if host else []),
    )


def _fetch_urlhaus() -> List[Case]:
    text = _http_get(URLHAUS_RECENT_CSV).text
    return [_urlhaus_row_to_case(r) for r in _parse_urlhaus_csv(text)]


# --- CISA KEV — known exploited vulnerabilities --------------------------

def _kev_vuln_to_case(v: Dict[str, object]) -> Case:
    cve = str(v["cveID"])
    vendor = str(v.get("vendorProject", ""))
    product = str(v.get("product", ""))
    vp = vendor if vendor == product else f"{vendor} {product}".strip()
    summary = f"CISA KEV: {v.get('vulnerabilityName', cve)}"
    if vp:
        summary += f" — {vp}"
    return Case(
        case_id=f"PSYC-KEV-{cve}",
        summary=summary,
        source_type="advisory",
        source_ref=f"https://nvd.nist.gov/vuln/detail/{cve}",
        source_metadata=dict(
            feed="cisa-kev", vendor=vendor, product=product,
            ransomware=str(v.get("knownRansomwareCampaignUse", "Unknown")),
            date_added=str(v.get("dateAdded", "")), due_date=str(v.get("dueDate", "")),
        ),
        observed_at=_parse_dt(str(v.get("dateAdded", "")), "%Y-%m-%d"),
        observables=Observables(cves=[cve]),
    )


def _fetch_cisa_kev() -> List[Case]:
    data = _http_get(CISA_KEV_JSON).json()
    return [_kev_vuln_to_case(v) for v in data.get("vulnerabilities", [])]


# --- Feodo Tracker — botnet C2 servers -----------------------------------

def _feodo_record_to_case(r: Dict[str, object]) -> Case:
    ip = str(r.get("ip_address", ""))
    port = str(r.get("port", ""))
    malware = str(r.get("malware", "botnet"))
    return Case(
        case_id=f"PSYC-FEODO-{ip}-{port}",
        summary=f"Feodo Tracker: {malware} botnet C2 at {ip}:{port}",
        source_type="abuse_feed",
        source_ref="https://feodotracker.abuse.ch/browse/",
        source_metadata=dict(
            feed="feodo", malware=malware, status=str(r.get("status", "")),
            port=port, country=str(r.get("country", "")), as_name=str(r.get("as_name", "")),
        ),
        observed_at=_parse_dt(str(r.get("first_seen", "")), "%Y-%m-%d %H:%M:%S"),
        observables=Observables(ips=[ip] if ip else []),
    )


def _fetch_feodo() -> List[Case]:
    data = _http_get(FEODO_BLOCKLIST_JSON).json()
    return [_feodo_record_to_case(r) for r in data]


# --- ThreatFox — multi-malware IOC feed (abuse.ch) -----------------------

# ThreatFox threat_type values → psyc IncidentType.
THREATFOX_THREAT_TYPE: Dict[str, IncidentType] = {
    "botnet_cc": IncidentType.BOTNET,
    "payload_delivery": IncidentType.MALWARE,
    "payload": IncidentType.MALWARE,
    "phishing": IncidentType.PHISHING,
}


def _threatfox_row_to_case(r: Dict[str, Any]) -> Optional[Case]:
    # API field is `ioc` (the `_value` alias is older docs); date is `first_seen`.
    ioc_value = str(r.get("ioc") or r.get("ioc_value") or "").strip()
    ioc_type = str(r.get("ioc_type") or "").lower()
    if not ioc_value or not ioc_type:
        return None
    malware = str(r.get("malware_printable") or r.get("malware") or "unknown")
    threat_type = str(r.get("threat_type") or "")
    tags_raw = r.get("tags") or []
    tags = tags_raw if isinstance(tags_raw, list) else []

    obs = Observables()
    host = ""
    if ioc_type in ("ip:port", "ipv4", "ipv6"):
        ip = ioc_value.split(":")[0]
        obs.ips = [ip]
    elif ioc_type == "domain":
        obs.domains = [ioc_value]
        host = ioc_value
    elif ioc_type == "url":
        obs.urls = [ioc_value]
        host = urlparse(ioc_value).hostname or ""
        if host:
            obs.domains = [host]
    elif ioc_type in ("sha256_hash", "md5_hash", "sha1_hash"):
        obs.hashes = [ioc_value]
    else:
        return None

    threat_label = threat_type.replace("_", " ") or "malware"
    summary = f"ThreatFox: {malware} {threat_label} — {ioc_value}"
    first_seen = str(r.get("first_seen") or r.get("first_seen_utc") or "")
    return Case(
        case_id=f"PSYC-THREATFOX-{r.get('id', '')}",
        summary=summary,
        source_type="abuse_feed",
        source_ref=str(r.get("reference") or f"https://threatfox.abuse.ch/ioc/{r.get('id', '')}/"),
        source_metadata=dict(
            feed="threatfox",
            malware=malware,
            malware_malpedia=str(r.get("malware_malpedia") or ""),
            threat_type=threat_type,
            threat_type_desc=str(r.get("threat_type_desc") or ""),
            ioc_type=ioc_type,
            confidence_level=str(r.get("confidence_level", "")),
            tags=",".join(t for t in tags if t),
            reporter=str(r.get("reporter", "")),
        ),
        observed_at=_parse_dt(first_seen, "%Y-%m-%d %H:%M:%S"),
        observables=obs,
    )


def _fetch_threatfox() -> List[Case]:
    key = os.environ.get("THREATFOX_AUTH_KEY", "").strip()
    if not key:
        raise RuntimeError("THREATFOX_AUTH_KEY not set — free abuse.ch auth-key from https://auth.abuse.ch/")
    data = _http("POST", THREATFOX_API, headers={"Auth-Key": key}, json_body={"query": "get_iocs", "days": 1}).json()
    rows = data.get("data") or []
    out: List[Case] = []
    for r in rows:
        c = _threatfox_row_to_case(r)
        if c is not None:
            out.append(c)
    return out


# --- MalwareBazaar — recent malware samples (abuse.ch) -------------------

def _mb_row_to_case(r: Dict[str, Any]) -> Optional[Case]:
    sha256 = str(r.get("sha256_hash") or "")
    if not sha256:
        return None
    sha1 = str(r.get("sha1_hash") or "")
    md5 = str(r.get("md5_hash") or "")
    file_name = str(r.get("file_name") or "unknown")
    signature = str(r.get("signature") or "")
    file_type = str(r.get("file_type") or "")
    tags_raw = r.get("tags") or []
    tags = tags_raw if isinstance(tags_raw, list) else []
    hashes = [h for h in (sha256, sha1, md5) if h]
    label = signature or "unsigned"
    summary = f"MalwareBazaar: {label} {file_type} sample — {file_name}"
    return Case(
        case_id=f"PSYC-MBAZAAR-{sha256[:16]}",
        summary=summary,
        source_type="abuse_feed",
        source_ref=f"https://bazaar.abuse.ch/sample/{sha256}/",
        source_metadata=dict(
            feed="malware-bazaar",
            signature=signature,
            file_type=file_type,
            file_name=file_name,
            tags=",".join(t for t in tags if t),
            reporter=str(r.get("reporter", "")),
        ),
        observed_at=_parse_dt(str(r.get("first_seen") or ""), "%Y-%m-%d %H:%M:%S"),
        observables=Observables(hashes=hashes),
    )


def _fetch_malware_bazaar() -> List[Case]:
    key = os.environ.get("THREATFOX_AUTH_KEY", "").strip()
    if not key:
        raise RuntimeError("THREATFOX_AUTH_KEY not set — abuse.ch auth-key from https://auth.abuse.ch/ also covers MalwareBazaar")
    # MalwareBazaar expects form-encoded body (unlike ThreatFox which takes JSON).
    data = _http("POST", MALWARE_BAZAAR_API, headers={"Auth-Key": key}, form_body={"query": "get_recent", "selector": "100"}).json()
    rows = data.get("data") or []
    out: List[Case] = []
    for r in rows:
        c = _mb_row_to_case(r)
        if c is not None:
            out.append(c)
    return out


# --- AlienVault OTX — curated multi-source pulses ------------------------

_OTX_IOC_LIMIT_PER_PULSE = 50


def _otx_pulse_to_case(p: Dict[str, Any]) -> Optional[Case]:
    pulse_id = str(p.get("id") or "")
    if not pulse_id:
        return None
    pulse_name = str(p.get("name") or "OTX pulse")
    description = str(p.get("description") or "")
    tags_raw = p.get("tags") or []
    tags = tags_raw if isinstance(tags_raw, list) else []
    tlp_pulse = str(p.get("tlp") or "white").upper()
    indicators = p.get("indicators") or []

    obs = Observables()
    for ind in indicators[:_OTX_IOC_LIMIT_PER_PULSE]:
        value = str(ind.get("indicator") or "").strip()
        itype = str(ind.get("type") or "").lower()
        if not value:
            continue
        if itype in ("ipv4", "ipv6"):
            obs.ips.append(value)
        elif itype in ("domain", "hostname"):
            obs.domains.append(value)
        elif itype == "url":
            obs.urls.append(value)
            host = urlparse(value).hostname or ""
            if host and host not in obs.domains:
                obs.domains.append(host)
        elif itype in ("filehash-sha256", "filehash-sha1", "filehash-md5"):
            obs.hashes.append(value)
        elif itype == "cve":
            obs.cves.append(value)

    if not (obs.urls or obs.domains or obs.ips or obs.hashes or obs.cves):
        return None

    return Case(
        case_id=f"PSYC-OTX-{pulse_id}",
        summary=f"OTX: {pulse_name}",
        source_type="threat_intel",
        source_ref=f"https://otx.alienvault.com/pulse/{pulse_id}",
        source_metadata=dict(
            feed="otx",
            pulse_name=pulse_name,
            description=description[:2000],
            tags=",".join(t for t in tags if t),
            tlp_pulse=tlp_pulse,
        ),
        observed_at=_parse_dt(str(p.get("created") or "").split(".")[0], "%Y-%m-%dT%H:%M:%S"),
        observables=obs,
    )


def _fetch_otx() -> List[Case]:
    key = os.environ.get("OTX_API_KEY", "").strip()
    if not key:
        raise RuntimeError("OTX_API_KEY not set — free key at https://otx.alienvault.com → settings → API")
    # OTX subscribes a new account to many curated feeds, so the unfiltered
    # /pulses/subscribed page can 504 on its own backend. modified_since
    # narrows to recent pulses; page size 20 caps the response.
    since = (datetime.now(timezone.utc) - timedelta(days=7)).strftime("%Y-%m-%dT%H:%M:%S")
    data = _http(
        "GET", OTX_PULSES_API,
        headers={"X-OTX-API-KEY": key},
        params={"limit": 20, "modified_since": since},
        timeout=120.0,
    ).json()
    pulses = data.get("results") or []
    out: List[Case] = []
    for p in pulses:
        c = _otx_pulse_to_case(p)
        if c is not None:
            out.append(c)
    return out


# --- registry + dispatch -------------------------------------------------

SOURCES: Dict[str, Callable[[], List[Case]]] = {
    "urlhaus": _fetch_urlhaus,
    "cisa-kev": _fetch_cisa_kev,
    "feodo": _fetch_feodo,
    "threatfox": _fetch_threatfox,
    "malware-bazaar": _fetch_malware_bazaar,
    "otx": _fetch_otx,
}


def fetch_and_signal(source: str, limit: Optional[int] = None) -> List[Case]:
    fetcher = SOURCES.get(source)
    if fetcher is None:
        raise ValueError(f"unknown source: {source}; known: {', '.join(SOURCES)}")
    cases = fetcher()
    if limit is not None:
        cases = cases[:limit]
    _log.info("scout.fetched", source=source, count=len(cases))
    return cases
