all repos — rastro @ 6533c8851b4a2970689b2081579d5f63f5136a58

BitTorrent tracker!

app/collectors/__init__.py (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
from __future__ import annotations

import logging

import httpx
from sqlalchemy.ext.asyncio import AsyncSession

from app.collectors.jokepool710 import Jokepool710Collector
from app.collectors.newtrackon import NewTrackonCollector
from app.collectors.trackerslist import TrackerslistCollector
from app.collectors.xiu2 import Xiu2Collector
from app.config import Settings, get_settings

logger = logging.getLogger(__name__)


async def import_all_sources(
    session: AsyncSession,
    settings: Settings | None = None,
    client: httpx.AsyncClient | None = None,
) -> dict[str, object]:
    settings = settings or get_settings()
    collectors = [
        TrackerslistCollector(),
        NewTrackonCollector(),
        Jokepool710Collector(),
        Xiu2Collector(),
    ]
    owns_client = client is None
    if client is None:
        client = httpx.AsyncClient(
            timeout=settings.probe_timeout_seconds,
            headers={"User-Agent": settings.user_agent()},
            follow_redirects=True,
        )
    results: dict[str, object] = {}
    try:
        for collector in collectors:
            stats = await collector.import_once(session, client, settings)
            results[collector.name] = stats
            logger.info(
                "import %s: received=%s valid=%s new=%s error=%s",
                collector.name,
                stats.received,
                stats.valid,
                stats.new,
                stats.error,
            )
    finally:
        if owns_client:
            await client.aclose()
    return results