from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass, field from datetime import UTC, datetime from typing import Protocol import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.config import Settings from app.db.models import CollectorCache, SourceImportRun, Tracker, TrackerSource from app.services.normalize import ( NormalizationError, NormalizedTracker, normalize_tracker_url, parse_tracker_list_text, ) @dataclass class ImportStats: received: int = 0 valid: int = 0 new: int = 0 unsupported: int = 0 skipped_invalid: int = 0 not_modified: bool = False error: str | None = None @dataclass class FeedFetchResult: body: str | None etag: str | None last_modified: str | None not_modified: bool = False status_code: int = 0 async def fetch_text_feed( client: httpx.AsyncClient, url: str, *, etag: str | None = None, last_modified: str | None = None, ) -> FeedFetchResult: headers: dict[str, str] = {} if etag: headers["If-None-Match"] = etag if last_modified: headers["If-Modified-Since"] = last_modified response = await client.get(url, headers=headers) if response.status_code == 304: return FeedFetchResult( body=None, etag=etag, last_modified=last_modified, not_modified=True, status_code=304, ) response.raise_for_status() return FeedFetchResult( body=response.text, etag=response.headers.get("ETag"), last_modified=response.headers.get("Last-Modified"), not_modified=False, status_code=response.status_code, ) async def get_cache(session: AsyncSession, feed_url: str) -> CollectorCache | None: result = await session.execute( select(CollectorCache).where(CollectorCache.feed_url == feed_url) ) return result.scalar_one_or_none() async def upsert_cache( session: AsyncSession, *, source_name: str, feed_url: str, etag: str | None, last_modified: str | None, ) -> None: row = await get_cache(session, feed_url) now = datetime.now(UTC) if row is None: session.add( CollectorCache( source_name=source_name, feed_url=feed_url, etag=etag, last_modified=last_modified, updated_at=now, ) ) else: row.etag = etag row.last_modified = last_modified row.updated_at = now def sanitize_error(message: str, limit: int = 512) -> str: text = " ".join(message.split()) return text[:limit] async def upsert_tracker_from_normalized( session: AsyncSession, normalized: NormalizedTracker, *, source_name: str, source_url: str, is_best: bool = False, is_stable: bool = False, is_live: bool = False, ) -> tuple[Tracker, bool]: """Insert or update tracker + source association. Returns (tracker, created).""" now = datetime.now(UTC) result = await session.execute( select(Tracker).where(Tracker.canonical_url == normalized.canonical_url) ) tracker = result.scalar_one_or_none() created = False if tracker is None: created = True tracker = Tracker( canonical_url=normalized.canonical_url, scheme=normalized.scheme, hostname=normalized.hostname, port=normalized.port, path=normalized.path, current_status="unsupported" if normalized.unsupported else "unknown", first_seen_at=now, last_seen_at=now, consecutive_failures=0, ) session.add(tracker) await session.flush() else: tracker.last_seen_at = now if normalized.unsupported: tracker.current_status = "unsupported" src_result = await session.execute( select(TrackerSource).where( TrackerSource.tracker_id == tracker.id, TrackerSource.source_name == source_name, ) ) association = src_result.scalar_one_or_none() if association is None: session.add( TrackerSource( tracker_id=tracker.id, source_name=source_name, source_url=source_url, is_best_at_source=is_best, is_stable_at_source=is_stable, is_live_at_source=is_live, first_seen_at=now, last_seen_at=now, ) ) else: association.last_seen_at = now association.source_url = source_url if is_best: association.is_best_at_source = True if is_stable: association.is_stable_at_source = True if is_live: association.is_live_at_source = True return tracker, created @dataclass class ParsedEntries: valid: list[NormalizedTracker] = field(default_factory=list) unsupported: list[NormalizedTracker] = field(default_factory=list) invalid: int = 0 received: int = 0 def parse_and_normalize(body: str) -> ParsedEntries: raw_urls = parse_tracker_list_text(body) parsed = ParsedEntries(received=len(raw_urls)) seen_canonical: set[str] = set() for raw in raw_urls: try: normalized = normalize_tracker_url(raw) except NormalizationError: parsed.invalid += 1 continue if normalized.canonical_url in seen_canonical: continue seen_canonical.add(normalized.canonical_url) if normalized.unsupported: parsed.unsupported.append(normalized) else: parsed.valid.append(normalized) return parsed @dataclass class EnrichmentFeed: """A secondary feed that only sets provenance flags (best/stable/live). Enrichment feeds are best-effort: a failure never fails the whole import and never touches measurement data. """ url: str is_best: bool = False is_stable: bool = False is_live: bool = False async def import_source_feeds( session: AsyncSession, client: httpx.AsyncClient, *, source_name: str, primary_url: str, enrichments: Sequence[EnrichmentFeed] = (), ) -> ImportStats: """Run the full import lifecycle for a text-feed source. Generalizes the pattern used by the trackerslist/newTrackon collectors: one primary feed (cached via ETag/Last-Modified, guarded against empty responses) plus zero or more best-effort enrichment feeds. """ stats = ImportStats() run = SourceImportRun( source_name=source_name, started_at=datetime.now(UTC), status="running", received_count=0, valid_count=0, new_count=0, unsupported_count=0, ) session.add(run) await session.flush() try: cache = await get_cache(session, primary_url) feed = await fetch_text_feed( client, primary_url, etag=cache.etag if cache else None, last_modified=cache.last_modified if cache else None, ) if feed.not_modified: stats.not_modified = True run.status = "not_modified" run.etag = feed.etag run.last_modified = feed.last_modified run.finished_at = datetime.now(UTC) await session.commit() return stats if not feed.body or not feed.body.strip(): # Empty body must not wipe existing data. run.status = "failed" run.error_message = sanitize_error(f"empty response from {source_name}") run.finished_at = datetime.now(UTC) stats.error = run.error_message await session.commit() return stats parsed = parse_and_normalize(feed.body) stats.received = parsed.received stats.unsupported = len(parsed.unsupported) stats.skipped_invalid = parsed.invalid for item in parsed.valid: _, created = await upsert_tracker_from_normalized( session, item, source_name=source_name, source_url=primary_url, ) stats.valid += 1 if created: stats.new += 1 for enrichment in enrichments: try: enrich_feed = await fetch_text_feed(client, enrichment.url) if not enrich_feed.body: continue for item in parse_and_normalize(enrich_feed.body).valid: await upsert_tracker_from_normalized( session, item, source_name=source_name, source_url=enrichment.url, is_best=enrichment.is_best, is_stable=enrichment.is_stable, is_live=enrichment.is_live, ) except Exception: # noqa: BLE001 — enrichment is best-effort continue await upsert_cache( session, source_name=source_name, feed_url=primary_url, etag=feed.etag, last_modified=feed.last_modified, ) run.received_count = stats.received run.valid_count = stats.valid run.new_count = stats.new run.unsupported_count = stats.unsupported run.status = "ok" run.etag = feed.etag run.last_modified = feed.last_modified run.finished_at = datetime.now(UTC) await session.commit() return stats except Exception as exc: # noqa: BLE001 await session.rollback() session.add( SourceImportRun( source_name=source_name, started_at=datetime.now(UTC), finished_at=datetime.now(UTC), status="failed", received_count=0, valid_count=0, new_count=0, unsupported_count=0, error_message=sanitize_error(str(exc)), ) ) await session.commit() stats.error = sanitize_error(str(exc)) return stats class Collector(Protocol): name: str async def import_once( self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings ) -> ImportStats: ...