all repos — rastro @ 6738696441c3bf8d4382193143952f0e7cdac6c5

BitTorrent tracker!

app/collectors/__init__.py (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
from __future__ import annotations

import logging

import httpx
from sqlalchemy.ext.asyncio import AsyncSession

from app.collectors.jokepool710 import Jokepool710Collector
from app.collectors.newtrackon import NewTrackonCollector
from app.collectors.trackerslist import TrackerslistCollector
from app.collectors.xiu2 import Xiu2Collector
from app.config import Settings, get_settings

logger = logging.getLogger(__name__)


async def import_all_sources(
    session: AsyncSession,
    settings: Settings | None = None,
    client: httpx.AsyncClient | None = None,
) -> dict[str, object]:
    settings = settings or get_settings()
    collectors = [
        TrackerslistCollector(),
        NewTrackonCollector(),
        Jokepool710Collector(),
        Xiu2Collector(),
    ]
    owns_client = client is None
    if client is None:
        client = httpx.AsyncClient(
            timeout=settings.probe_timeout_seconds,
            headers={"User-Agent": settings.user_agent()},
            follow_redirects=True,
        )
    results: dict[str, object] = {}
    try:
        for collector in collectors:
            stats = await collector.import_once(session, client, settings)
            results[collector.name] = stats
            logger.info(
                "import %s: received=%s valid=%s new=%s error=%s",
                collector.name,
                stats.received,
                stats.valid,
                stats.new,
                stats.error,
            )
    finally:
        if owns_client:
            await client.aclose()
    return results