all repos — rastro @ 303f75c7f98bcf857e2ef49ec4c4489ce9eb333b

BitTorrent tracker!

app/collectors/__init__.py (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
from __future__ import annotations

import logging

import httpx
from sqlalchemy.ext.asyncio import AsyncSession

from app.collectors.jokepool710 import Jokepool710Collector
from app.collectors.newtrackon import NewTrackonCollector
from app.collectors.trackerslist import TrackerslistCollector
from app.collectors.xiu2 import Xiu2Collector
from app.config import Settings, get_settings

logger = logging.getLogger(__name__)


async def import_all_sources(
    session: AsyncSession,
    settings: Settings | None = None,
    client: httpx.AsyncClient | None = None,
) -> dict[str, object]:
    settings = settings or get_settings()
    collectors = [
        TrackerslistCollector(),
        NewTrackonCollector(),
        Jokepool710Collector(),
        Xiu2Collector(),
    ]
    owns_client = client is None
    if client is None:
        client = httpx.AsyncClient(
            timeout=settings.probe_timeout_seconds,
            headers={"User-Agent": settings.user_agent()},
            follow_redirects=True,
        )
    results: dict[str, object] = {}
    try:
        for collector in collectors:
            stats = await collector.import_once(session, client, settings)
            results[collector.name] = stats
            logger.info(
                "import %s: received=%s valid=%s new=%s error=%s",
                collector.name,
                stats.received,
                stats.valid,
                stats.new,
                stats.error,
            )
    finally:
        if owns_client:
            await client.aclose()
    return results