from __future__ import annotations import logging import httpx from sqlalchemy.ext.asyncio import AsyncSession from app.collectors.jokepool710 import Jokepool710Collector from app.collectors.newtrackon import NewTrackonCollector from app.collectors.trackerslist import TrackerslistCollector from app.collectors.xiu2 import Xiu2Collector from app.config import Settings, get_settings logger = logging.getLogger(__name__) async def import_all_sources( session: AsyncSession, settings: Settings | None = None, client: httpx.AsyncClient | None = None, ) -> dict[str, object]: settings = settings or get_settings() collectors = [ TrackerslistCollector(), NewTrackonCollector(), Jokepool710Collector(), Xiu2Collector(), ] owns_client = client is None if client is None: client = httpx.AsyncClient( timeout=settings.probe_timeout_seconds, headers={"User-Agent": settings.user_agent()}, follow_redirects=True, ) results: dict[str, object] = {} try: for collector in collectors: stats = await collector.import_once(session, client, settings) results[collector.name] = stats logger.info( "import %s: received=%s valid=%s new=%s error=%s", collector.name, stats.received, stats.valid, stats.new, stats.error, ) finally: if owns_client: await client.aclose() return results