app/collectors/__init__.py (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
from __future__ import annotations
import logging
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from app.collectors.jokepool710 import Jokepool710Collector
from app.collectors.newtrackon import NewTrackonCollector
from app.collectors.trackerslist import TrackerslistCollector
from app.collectors.xiu2 import Xiu2Collector
from app.config import Settings, get_settings
logger = logging.getLogger(__name__)
async def import_all_sources(
session: AsyncSession,
settings: Settings | None = None,
client: httpx.AsyncClient | None = None,
) -> dict[str, object]:
settings = settings or get_settings()
collectors = [
TrackerslistCollector(),
NewTrackonCollector(),
Jokepool710Collector(),
Xiu2Collector(),
]
owns_client = client is None
if client is None:
client = httpx.AsyncClient(
timeout=settings.probe_timeout_seconds,
headers={"User-Agent": settings.user_agent()},
follow_redirects=True,
)
results: dict[str, object] = {}
try:
for collector in collectors:
stats = await collector.import_once(session, client, settings)
results[collector.name] = stats
logger.info(
"import %s: received=%s valid=%s new=%s error=%s",
collector.name,
stats.received,
stats.valid,
stats.new,
stats.error,
)
finally:
if owns_client:
await client.aclose()
return results
|