init: scaffold psyc — defensive CTI routing & evidence-sealing platform
Stage-1 vertical slice: Pydantic Case model, SQLAlchemy Core persistence, URLhaus Scoutline fetcher, FastAPI/Jinja cockpit (cases list + detail), flat Typer CLI, Result[T, E] type module, structlog config. Architecture in docs/dossier.md; 12-fold style guide in docs/style.md. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
1
src/psyc/__init__.py
Normal file
1
src/psyc/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__version__ = "0.1.0"
|
||||
45
src/psyc/cli.py
Normal file
45
src/psyc/cli.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""psyc CLI — flat Typer commands, hyphenated names."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typer
|
||||
import uvicorn
|
||||
|
||||
from psyc import db, log
|
||||
from psyc.lines import scout
|
||||
|
||||
|
||||
app = typer.Typer(add_completion=False, help="psyc — defensive CTI routing & sealing")
|
||||
log.configure()
|
||||
_log = log.get(__name__)
|
||||
|
||||
|
||||
@app.command("init")
|
||||
def init() -> None:
|
||||
path = db.init_db()
|
||||
typer.echo(f"db ready @ {path}")
|
||||
|
||||
|
||||
@app.command("status")
|
||||
def status() -> None:
|
||||
typer.echo(f"cases: {db.case_count()}")
|
||||
|
||||
|
||||
@app.command("fetch-urlhaus")
|
||||
def fetch_urlhaus(limit: int = typer.Option(50, help="max rows to ingest from the feed")) -> None:
|
||||
db.init_db()
|
||||
typer.echo(f"fetching URLhaus recent feed (limit={limit})…")
|
||||
cases = scout.fetch_and_signal(limit=limit)
|
||||
for c in cases:
|
||||
db.upsert_case(c)
|
||||
typer.echo(f"ingested {len(cases)} case(s). total now: {db.case_count()}")
|
||||
|
||||
|
||||
@app.command("serve")
|
||||
def serve(host: str = "127.0.0.1", port: int = 8000, reload: bool = False) -> None:
|
||||
db.init_db()
|
||||
uvicorn.run("psyc.cockpit.app:app", host=host, port=port, reload=reload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
0
src/psyc/cockpit/__init__.py
Normal file
0
src/psyc/cockpit/__init__.py
Normal file
49
src/psyc/cockpit/app.py
Normal file
49
src/psyc/cockpit/app.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""psyc Operations Cockpit — FastAPI + Jinja."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from psyc import db, log
|
||||
from psyc.result import Err
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
TEMPLATES = Jinja2Templates(directory=str(HERE / "templates"))
|
||||
|
||||
log.configure()
|
||||
_log = log.get(__name__)
|
||||
|
||||
app = FastAPI(title="psyc Operations Cockpit", version="0.1.0")
|
||||
app.mount("/static", StaticFiles(directory=str(HERE / "static")), name="static")
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index(request: Request) -> HTMLResponse:
|
||||
cases = db.list_cases(limit=200)
|
||||
total = db.case_count()
|
||||
return TEMPLATES.TemplateResponse(request, "cases.html", {"cases": cases, "total": total})
|
||||
|
||||
|
||||
@app.get("/cases", response_class=HTMLResponse)
|
||||
def cases_list(request: Request) -> HTMLResponse:
|
||||
return index(request)
|
||||
|
||||
|
||||
@app.get("/cases/{case_id}", response_class=HTMLResponse)
|
||||
def case_detail(request: Request, case_id: str) -> HTMLResponse:
|
||||
result = db.get_case(case_id)
|
||||
if isinstance(result, Err):
|
||||
_log.info("cockpit.case.miss", case_id=case_id)
|
||||
raise HTTPException(status_code=404, detail=result.reason)
|
||||
return TEMPLATES.TemplateResponse(request, "case_detail.html", {"case": result.value})
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict:
|
||||
return {"status": "ok"}
|
||||
96
src/psyc/cockpit/static/cockpit.css
Normal file
96
src/psyc/cockpit/static/cockpit.css
Normal file
@@ -0,0 +1,96 @@
|
||||
:root {
|
||||
--bg: #0f1115;
|
||||
--panel: #161a22;
|
||||
--panel-2: #1c2230;
|
||||
--border: #262d3a;
|
||||
--text: #d8dee9;
|
||||
--muted: #7d8597;
|
||||
--accent: #1ec8ff;
|
||||
--accent-glow: rgba(30, 200, 255, 0.25);
|
||||
--green: #4ade80;
|
||||
--amber: #fbbf24;
|
||||
--red: #f87171;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-size: 14px;
|
||||
line-height: 1.45;
|
||||
}
|
||||
a { color: #93c5fd; text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
code { background: var(--panel-2); padding: 1px 5px; border-radius: 3px; }
|
||||
|
||||
.topbar {
|
||||
display: flex; justify-content: space-between; align-items: center;
|
||||
padding: 10px 24px;
|
||||
background: var(--panel);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.brand {
|
||||
display: flex; align-items: center; gap: 12px;
|
||||
color: inherit; text-decoration: none;
|
||||
}
|
||||
.brand:hover { text-decoration: none; }
|
||||
.brand-icon {
|
||||
height: 48px; width: 160px;
|
||||
object-fit: cover; object-position: center;
|
||||
display: block;
|
||||
filter: drop-shadow(0 0 8px var(--accent-glow));
|
||||
}
|
||||
.brand-sub {
|
||||
color: var(--muted); font-size: 11px;
|
||||
text-transform: uppercase; letter-spacing: 1.2px;
|
||||
padding-left: 12px; border-left: 1px solid var(--border);
|
||||
}
|
||||
.nav a { margin-left: 18px; color: var(--muted); }
|
||||
.nav a:hover { color: var(--text); }
|
||||
|
||||
.content { padding: 24px; max-width: 1280px; margin: 0 auto; }
|
||||
.footer { text-align: center; color: var(--muted); padding: 24px; font-size: 12px; }
|
||||
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
padding: 20px 24px;
|
||||
}
|
||||
.panel-head { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 16px; }
|
||||
.panel-head h1 { margin: 0; font-size: 18px; }
|
||||
.count { color: var(--muted); font-size: 12px; }
|
||||
.empty { color: var(--muted); }
|
||||
.back { color: var(--muted); font-size: 12px; }
|
||||
.summary-lead { color: var(--text); font-size: 15px; margin-top: 8px; }
|
||||
|
||||
table.cases { width: 100%; border-collapse: collapse; }
|
||||
table.cases th, table.cases td { text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border); }
|
||||
table.cases th { color: var(--muted); font-weight: 500; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px; }
|
||||
table.cases tr:hover { background: var(--panel-2); }
|
||||
table.cases .muted { color: var(--muted); font-size: 12px; }
|
||||
table.cases .summary { max-width: 600px; }
|
||||
|
||||
.sev-badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; background: var(--panel-2); border: 1px solid var(--border); text-transform: uppercase; }
|
||||
tr.sev-critical .sev-badge { color: var(--red); border-color: var(--red); }
|
||||
tr.sev-high .sev-badge { color: var(--amber); border-color: var(--amber); }
|
||||
tr.sev-medium .sev-badge { color: #fde68a; }
|
||||
tr.sev-low .sev-badge { color: var(--muted); }
|
||||
|
||||
.tlp-badge { display: inline-block; padding: 2px 8px; border-radius: 3px; font-size: 11px; font-weight: 600; }
|
||||
.tlp-RED { background: #7f1d1d; color: #fee2e2; }
|
||||
.tlp-AMBER { background: #78350f; color: #fef3c7; }
|
||||
.tlp-GREEN { background: #14532d; color: #d1fae5; }
|
||||
.tlp-CLEAR { background: var(--panel-2); color: var(--muted); border: 1px solid var(--border); }
|
||||
|
||||
.grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; margin-top: 20px; }
|
||||
.card { background: var(--panel-2); border: 1px solid var(--border); border-radius: 6px; padding: 14px 16px; }
|
||||
.card.wide { grid-column: 1 / -1; }
|
||||
.card h2 { font-size: 13px; color: var(--muted); margin: 0 0 10px; text-transform: uppercase; letter-spacing: 1px; }
|
||||
.card h3 { font-size: 12px; color: var(--muted); margin: 12px 0 6px; }
|
||||
.card dl { display: grid; grid-template-columns: 140px 1fr; gap: 6px 12px; margin: 0; font-size: 13px; }
|
||||
.card dt { color: var(--muted); }
|
||||
.card dd { margin: 0; }
|
||||
.card ul { margin: 0; padding-left: 18px; font-size: 13px; }
|
||||
BIN
src/psyc/cockpit/static/psyc-logo.png
Normal file
BIN
src/psyc/cockpit/static/psyc-logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 857 KiB |
25
src/psyc/cockpit/templates/base.html
Normal file
25
src/psyc/cockpit/templates/base.html
Normal file
@@ -0,0 +1,25 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>{% block title %}psyc cockpit{% endblock %}</title>
|
||||
<link rel="icon" type="image/png" href="/static/psyc-logo.png">
|
||||
<link rel="stylesheet" href="/static/cockpit.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<a class="brand" href="/cases">
|
||||
<img class="brand-icon" src="/static/psyc-logo.png" alt="psyc">
|
||||
<span class="brand-sub">operations cockpit</span>
|
||||
</a>
|
||||
<nav class="nav">
|
||||
<a href="/cases">Cases</a>
|
||||
<a href="/healthz">Health</a>
|
||||
</nav>
|
||||
</header>
|
||||
<main class="content">
|
||||
{% block content %}{% endblock %}
|
||||
</main>
|
||||
<footer class="footer">psyc 0.1 — defensive CTI prototype</footer>
|
||||
</body>
|
||||
</html>
|
||||
50
src/psyc/cockpit/templates/case_detail.html
Normal file
50
src/psyc/cockpit/templates/case_detail.html
Normal file
@@ -0,0 +1,50 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ case.case_id }} — psyc{% endblock %}
|
||||
{% block content %}
|
||||
<section class="panel">
|
||||
<a class="back" href="/cases">← back to cases</a>
|
||||
<h1>{{ case.case_id }}</h1>
|
||||
<p class="summary-lead">{{ case.summary }}</p>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
<h2>Classification</h2>
|
||||
<dl>
|
||||
<dt>Severity</dt><dd>{{ case.classification.severity.value if case.classification.severity else '—' }}</dd>
|
||||
<dt>TLP</dt><dd><span class="tlp-badge tlp-{{ case.classification.tlp.value }}">{{ case.classification.tlp.value }}</span></dd>
|
||||
<dt>Incident type</dt><dd>{{ case.classification.incident_type.value if case.classification.incident_type else '—' }}</dd>
|
||||
<dt>Internal class</dt><dd>{{ case.classification.internal_class.value if case.classification.internal_class else '—' }}</dd>
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Confidence</h2>
|
||||
<dl>
|
||||
<dt>Level</dt><dd>{{ case.confidence.level }}</dd>
|
||||
<dt>Source reliability</dt><dd>{{ case.confidence.source_reliability }}</dd>
|
||||
<dt>Information credibility</dt><dd>{{ case.confidence.information_credibility }}</dd>
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Source</h2>
|
||||
<dl>
|
||||
<dt>Type</dt><dd>{{ case.source_type }}</dd>
|
||||
<dt>Reference</dt><dd>{% if case.source_ref %}<a href="{{ case.source_ref }}" target="_blank" rel="noopener">{{ case.source_ref }}</a>{% else %}—{% endif %}</dd>
|
||||
<dt>Observed</dt><dd class="muted">{{ case.observed_at.strftime('%Y-%m-%d %H:%M UTC') }}</dd>
|
||||
<dt>Ingested</dt><dd class="muted">{{ case.ingested_at.strftime('%Y-%m-%d %H:%M UTC') }}</dd>
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<div class="card wide">
|
||||
<h2>Observables</h2>
|
||||
{% set obs = case.observables %}
|
||||
{% if obs.urls %}<h3>URLs</h3><ul>{% for u in obs.urls %}<li><code>{{ u }}</code></li>{% endfor %}</ul>{% endif %}
|
||||
{% if obs.domains %}<h3>Domains</h3><ul>{% for d in obs.domains %}<li><code>{{ d }}</code></li>{% endfor %}</ul>{% endif %}
|
||||
{% if obs.ips %}<h3>IPs</h3><ul>{% for i in obs.ips %}<li><code>{{ i }}</code></li>{% endfor %}</ul>{% endif %}
|
||||
{% if obs.hashes %}<h3>Hashes</h3><ul>{% for h in obs.hashes %}<li><code>{{ h }}</code></li>{% endfor %}</ul>{% endif %}
|
||||
{% if not (obs.urls or obs.domains or obs.ips or obs.hashes) %}<p class="muted">no observables</p>{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
38
src/psyc/cockpit/templates/cases.html
Normal file
38
src/psyc/cockpit/templates/cases.html
Normal file
@@ -0,0 +1,38 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Cases — psyc{% endblock %}
|
||||
{% block content %}
|
||||
<section class="panel">
|
||||
<div class="panel-head">
|
||||
<h1>Case Queue</h1>
|
||||
<span class="count">{{ total }} case{{ '' if total == 1 else 's' }}</span>
|
||||
</div>
|
||||
{% if not cases %}
|
||||
<p class="empty">No cases yet. Run <code>psyc fetch urlhaus</code> to ingest.</p>
|
||||
{% else %}
|
||||
<table class="cases">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Case ID</th>
|
||||
<th>Ingested</th>
|
||||
<th>Severity</th>
|
||||
<th>TLP</th>
|
||||
<th>Type</th>
|
||||
<th>Summary</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for c in cases %}
|
||||
<tr class="sev-{{ c.classification.severity.value if c.classification.severity else 'none' }}">
|
||||
<td><a href="/cases/{{ c.case_id }}">{{ c.case_id }}</a></td>
|
||||
<td class="muted">{{ c.ingested_at.strftime('%Y-%m-%d %H:%M') }}</td>
|
||||
<td><span class="sev-badge">{{ c.classification.severity.value if c.classification.severity else '—' }}</span></td>
|
||||
<td><span class="tlp-badge tlp-{{ c.classification.tlp.value }}">{{ c.classification.tlp.value }}</span></td>
|
||||
<td>{{ c.classification.incident_type.value if c.classification.incident_type else '—' }}</td>
|
||||
<td class="summary">{{ c.summary }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endblock %}
|
||||
135
src/psyc/db.py
Normal file
135
src/psyc/db.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""SQLite persistence — case store + append-only ledger. SQLAlchemy Core."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Column,
|
||||
DateTime,
|
||||
Engine,
|
||||
Index,
|
||||
Integer,
|
||||
MetaData,
|
||||
String,
|
||||
Table,
|
||||
Text,
|
||||
create_engine,
|
||||
func,
|
||||
insert,
|
||||
select,
|
||||
)
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
|
||||
from psyc import log
|
||||
from psyc.models import Case
|
||||
from psyc.result import Err, Ok, Result
|
||||
|
||||
|
||||
DB_PATH = Path(__file__).resolve().parent.parent.parent / "data" / "psyc.db"
|
||||
|
||||
_metadata = MetaData()
|
||||
|
||||
cases = Table(
|
||||
"cases", _metadata,
|
||||
Column("case_id", String, primary_key=True),
|
||||
Column("summary", Text, nullable=False),
|
||||
Column("source_type", String, nullable=False),
|
||||
Column("source_ref", Text, nullable=False),
|
||||
Column("observed_at", String, nullable=False),
|
||||
Column("ingested_at", String, nullable=False),
|
||||
Column("tlp", String, nullable=False),
|
||||
Column("severity", String, nullable=True),
|
||||
Column("incident_type", String, nullable=True),
|
||||
Column("payload", Text, nullable=False),
|
||||
)
|
||||
Index("cases_ingested_idx", cases.c.ingested_at.desc())
|
||||
Index("cases_severity_idx", cases.c.severity)
|
||||
|
||||
ledger = Table(
|
||||
"ledger", _metadata,
|
||||
Column("id", Integer, primary_key=True, autoincrement=True),
|
||||
Column("timestamp", String, nullable=False),
|
||||
Column("case_id", String, nullable=False),
|
||||
Column("destination", String, nullable=False),
|
||||
Column("payload_hash", String, nullable=False),
|
||||
Column("submitter_identity", String, nullable=False),
|
||||
Column("tlp", String, nullable=False),
|
||||
Column("response_id", String, nullable=True),
|
||||
Column("outcome", String, nullable=False),
|
||||
Column("detail", Text, nullable=True),
|
||||
)
|
||||
Index("ledger_case_idx", ledger.c.case_id)
|
||||
Index("ledger_time_idx", ledger.c.timestamp.desc())
|
||||
|
||||
|
||||
_log = log.get(__name__)
|
||||
_engine: Optional[Engine] = None
|
||||
|
||||
|
||||
def engine(db_path: Path = DB_PATH) -> Engine:
|
||||
global _engine
|
||||
if _engine is None:
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
_engine = create_engine(f"sqlite:///{db_path}", future=True)
|
||||
return _engine
|
||||
|
||||
|
||||
def init_db(db_path: Path = DB_PATH) -> Path:
|
||||
eng = engine(db_path)
|
||||
_metadata.create_all(eng, checkfirst=True)
|
||||
_log.info("db.ready", path=str(db_path))
|
||||
return db_path
|
||||
|
||||
|
||||
def upsert_case(case: Case, db_path: Path = DB_PATH) -> None:
|
||||
payload = case.model_dump_json()
|
||||
values = dict(
|
||||
case_id=case.case_id,
|
||||
summary=case.summary,
|
||||
source_type=case.source_type,
|
||||
source_ref=case.source_ref,
|
||||
observed_at=case.observed_at.isoformat(),
|
||||
ingested_at=case.ingested_at.isoformat(),
|
||||
tlp=case.classification.tlp.value,
|
||||
severity=case.classification.severity.value if case.classification.severity else None,
|
||||
incident_type=case.classification.incident_type.value if case.classification.incident_type else None,
|
||||
payload=payload,
|
||||
)
|
||||
stmt = sqlite_insert(cases).values(**values)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[cases.c.case_id],
|
||||
set_=dict(
|
||||
summary=stmt.excluded.summary,
|
||||
tlp=stmt.excluded.tlp,
|
||||
severity=stmt.excluded.severity,
|
||||
incident_type=stmt.excluded.incident_type,
|
||||
payload=stmt.excluded.payload,
|
||||
),
|
||||
)
|
||||
with engine(db_path).begin() as conn:
|
||||
conn.execute(stmt)
|
||||
|
||||
|
||||
def list_cases(limit: int = 100, db_path: Path = DB_PATH) -> List[Case]:
|
||||
stmt = select(cases.c.payload).order_by(cases.c.ingested_at.desc()).limit(limit)
|
||||
with engine(db_path).connect() as conn:
|
||||
rows = conn.execute(stmt).fetchall()
|
||||
return [Case.model_validate_json(r.payload) for r in rows]
|
||||
|
||||
|
||||
def get_case(case_id: str, db_path: Path = DB_PATH) -> Result[Case, str]:
|
||||
stmt = select(cases.c.payload).where(cases.c.case_id == case_id)
|
||||
with engine(db_path).connect() as conn:
|
||||
row = conn.execute(stmt).fetchone()
|
||||
if row is None:
|
||||
return Err(f"case not found: {case_id}")
|
||||
return Ok(Case.model_validate_json(row.payload))
|
||||
|
||||
|
||||
def case_count(db_path: Path = DB_PATH) -> int:
|
||||
stmt = select(func.count()).select_from(cases)
|
||||
with engine(db_path).connect() as conn:
|
||||
return conn.execute(stmt).scalar_one()
|
||||
0
src/psyc/lines/__init__.py
Normal file
0
src/psyc/lines/__init__.py
Normal file
108
src/psyc/lines/scout.py
Normal file
108
src/psyc/lines/scout.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Scoutline — Fetcher + Signalizer for URLhaus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from psyc import log
|
||||
from psyc.models import (
|
||||
Case,
|
||||
Classification,
|
||||
Confidence,
|
||||
IncidentType,
|
||||
Observables,
|
||||
Severity,
|
||||
TLP,
|
||||
)
|
||||
|
||||
|
||||
URLHAUS_RECENT_CSV = "https://urlhaus.abuse.ch/downloads/csv_recent/"
|
||||
USER_AGENT = "psyc/0.1 (defensive CTI; hackathon prototype)"
|
||||
|
||||
_log = log.get(__name__)
|
||||
|
||||
|
||||
def fetch_recent_csv(timeout: float = 30.0) -> str:
|
||||
with httpx.Client(timeout=timeout, headers={"User-Agent": USER_AGENT}) as client:
|
||||
resp = client.get(URLHAUS_RECENT_CSV)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
|
||||
def _parse_urlhaus_date(s: str) -> datetime:
|
||||
try:
|
||||
return datetime.strptime(s, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def parse_urlhaus_csv(csv_text: str) -> Iterable[Dict[str, str]]:
|
||||
lines = [ln for ln in csv_text.splitlines() if ln and not ln.startswith("#")]
|
||||
if not lines:
|
||||
return
|
||||
reader = csv.reader(io.StringIO("\n".join(lines)), quotechar='"')
|
||||
for fields in reader:
|
||||
if len(fields) < 9:
|
||||
continue
|
||||
yield {
|
||||
"id": fields[0],
|
||||
"dateadded": fields[1],
|
||||
"url": fields[2],
|
||||
"url_status": fields[3],
|
||||
"last_online": fields[4],
|
||||
"threat": fields[5],
|
||||
"tags": fields[6],
|
||||
"urlhaus_link": fields[7],
|
||||
"reporter": fields[8],
|
||||
}
|
||||
|
||||
|
||||
def row_to_case(row: Dict[str, str]) -> Case:
|
||||
url = row["url"]
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or ""
|
||||
tags = [t.strip() for t in row["tags"].split(",") if t.strip()]
|
||||
|
||||
severity = Severity.HIGH if row["url_status"] == "online" else Severity.MEDIUM
|
||||
classification = Classification(
|
||||
severity=severity,
|
||||
tlp=TLP.GREEN,
|
||||
incident_type=IncidentType.MALWARE,
|
||||
)
|
||||
confidence = Confidence(
|
||||
level="medium",
|
||||
source_reliability="B",
|
||||
information_credibility="2",
|
||||
)
|
||||
observables = Observables(urls=[url], domains=[host] if host else [])
|
||||
summary = f"URLhaus: {row['threat'] or 'malware_distribution'} at {host or url}"
|
||||
if tags:
|
||||
summary += f" (tags: {', '.join(tags[:4])})"
|
||||
|
||||
return Case(
|
||||
case_id=f"PSYC-URLHAUS-{row['id']}",
|
||||
summary=summary,
|
||||
source_type="abuse_feed",
|
||||
source_ref=row["urlhaus_link"],
|
||||
observed_at=_parse_urlhaus_date(row["dateadded"]),
|
||||
classification=classification,
|
||||
confidence=confidence,
|
||||
observables=observables,
|
||||
)
|
||||
|
||||
|
||||
def fetch_and_signal(limit: Optional[int] = None) -> List[Case]:
|
||||
csv_text = fetch_recent_csv()
|
||||
cases: List[Case] = []
|
||||
for row in parse_urlhaus_csv(csv_text):
|
||||
cases.append(row_to_case(row))
|
||||
if limit is not None and len(cases) >= limit:
|
||||
break
|
||||
_log.info("scout.urlhaus.fetched", count=len(cases))
|
||||
return cases
|
||||
45
src/psyc/log.py
Normal file
45
src/psyc/log.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""structlog configuration — call configure() once at process start."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import structlog
|
||||
|
||||
|
||||
_configured = False
|
||||
|
||||
|
||||
def configure(level: str = "INFO", json: bool = False) -> None:
|
||||
global _configured
|
||||
if _configured:
|
||||
return
|
||||
|
||||
logging.basicConfig(format="%(message)s", stream=sys.stderr, level=level)
|
||||
|
||||
renderer = (
|
||||
structlog.processors.JSONRenderer()
|
||||
if json
|
||||
else structlog.dev.ConsoleRenderer(colors=sys.stderr.isatty())
|
||||
)
|
||||
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.processors.add_log_level,
|
||||
structlog.processors.TimeStamper(fmt="iso", utc=True),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
structlog.processors.format_exc_info,
|
||||
renderer,
|
||||
],
|
||||
wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, level)),
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
_configured = True
|
||||
|
||||
|
||||
def get(name: str) -> structlog.stdlib.BoundLogger:
|
||||
if not _configured:
|
||||
configure()
|
||||
return structlog.get_logger(name)
|
||||
115
src/psyc/models.py
Normal file
115
src/psyc/models.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""Normalized case object — the contract between every worker line."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class TLP(str, Enum):
|
||||
RED = "RED"
|
||||
AMBER = "AMBER"
|
||||
GREEN = "GREEN"
|
||||
CLEAR = "CLEAR"
|
||||
|
||||
|
||||
class Severity(str, Enum):
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class InternalClass(str, Enum):
|
||||
A = "A"
|
||||
B = "B"
|
||||
C = "C"
|
||||
D = "D"
|
||||
E = "E"
|
||||
|
||||
|
||||
class IncidentType(str, Enum):
|
||||
ACCESS_SALE = "access_sale"
|
||||
RANSOMWARE = "ransomware"
|
||||
CREDENTIAL_LEAK = "credential_leak"
|
||||
PHISHING = "phishing"
|
||||
MALWARE = "malware"
|
||||
EXPLOIT = "exploit"
|
||||
BOTNET = "botnet"
|
||||
DATA_LEAK = "data_leak"
|
||||
|
||||
|
||||
class Classification(BaseModel):
|
||||
internal_class: Optional[InternalClass] = None
|
||||
severity: Optional[Severity] = None
|
||||
tlp: TLP = TLP.AMBER
|
||||
incident_type: Optional[IncidentType] = None
|
||||
|
||||
|
||||
class Confidence(BaseModel):
|
||||
level: str = "low"
|
||||
source_reliability: str = "unknown"
|
||||
information_credibility: str = "unknown"
|
||||
|
||||
|
||||
class Victim(BaseModel):
|
||||
name: str = ""
|
||||
domain: str = ""
|
||||
country: str = ""
|
||||
sector: str = ""
|
||||
critical_infrastructure: bool = False
|
||||
|
||||
|
||||
class Actor(BaseModel):
|
||||
name: str = ""
|
||||
aliases: List[str] = Field(default_factory=list)
|
||||
campaign: str = ""
|
||||
confidence: str = "low"
|
||||
|
||||
|
||||
class Observables(BaseModel):
|
||||
domains: List[str] = Field(default_factory=list)
|
||||
ips: List[str] = Field(default_factory=list)
|
||||
urls: List[str] = Field(default_factory=list)
|
||||
hashes: List[str] = Field(default_factory=list)
|
||||
cves: List[str] = Field(default_factory=list)
|
||||
wallets: List[str] = Field(default_factory=list)
|
||||
emails: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class Evidence(BaseModel):
|
||||
raw_evidence_location: str = ""
|
||||
sealed_package_id: str = ""
|
||||
payload_hash: str = ""
|
||||
plaintext_destroyed: bool = False
|
||||
local_unwrapped_key_destroyed: bool = False
|
||||
|
||||
|
||||
class Routing(BaseModel):
|
||||
recommended_routes: List[str] = Field(default_factory=list)
|
||||
blocked_routes: List[str] = Field(default_factory=list)
|
||||
human_approval_required: bool = True
|
||||
|
||||
|
||||
class Case(BaseModel):
|
||||
case_id: str = Field(default_factory=lambda: f"PSYC-{uuid4().hex[:12].upper()}")
|
||||
summary: str = ""
|
||||
source_type: str = ""
|
||||
source_ref: str = ""
|
||||
observed_at: datetime = Field(default_factory=utcnow)
|
||||
ingested_at: datetime = Field(default_factory=utcnow)
|
||||
classification: Classification = Field(default_factory=Classification)
|
||||
confidence: Confidence = Field(default_factory=Confidence)
|
||||
victim: Victim = Field(default_factory=Victim)
|
||||
actor: Actor = Field(default_factory=Actor)
|
||||
observables: Observables = Field(default_factory=Observables)
|
||||
evidence: Evidence = Field(default_factory=Evidence)
|
||||
routing: Routing = Field(default_factory=Routing)
|
||||
31
src/psyc/result.py
Normal file
31
src/psyc/result.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""Result types — Ok/Err for explicit success/failure return values."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Generic, TypeVar, Union
|
||||
|
||||
T = TypeVar("T")
|
||||
E = TypeVar("E")
|
||||
|
||||
|
||||
class Ok(Generic[T]):
|
||||
__slots__ = ("value",)
|
||||
|
||||
def __init__(self, value: T) -> None:
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Ok({self.value!r})"
|
||||
|
||||
|
||||
class Err(Generic[E]):
|
||||
__slots__ = ("reason",)
|
||||
|
||||
def __init__(self, reason: E) -> None:
|
||||
self.reason = reason
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Err({self.reason!r})"
|
||||
|
||||
|
||||
Result = Union[Ok[T], Err[E]]
|
||||
Reference in New Issue
Block a user