first commit
jump to
@@ -0,0 +1,34 @@
+# Keep the build context small and fast. +.git +.venv +venv + +# Python caches / build artifacts +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +dist/ +build/ + +# Tooling caches +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +# Local state and secrets (never baked into the image) +.env +data/*.db +data/*.db-* + +# Tests are not needed at runtime +tests/ + +# IDE / OS +.idea/ +.vscode/ +*.swp +.DS_Store +Thumbs.db
@@ -0,0 +1,36 @@
+APP_NAME=Rastro +APP_DESCRIPTION=Observatorio publico de trackers BitTorrent +APP_VERSION=0.1.0 +PUBLIC_URL=http://localhost:8090 +REPOSITORY_URL= +DATABASE_URL=sqlite+aiosqlite:///data/rastro.db +HOST=0.0.0.0 +PORT=8090 +LOG_LEVEL=INFO +RUN_SCHEDULER=true +# Atualizacao semanal das listas (cron em UTC). Dia: mon..sun ou 0..6. +IMPORT_DAY_OF_WEEK=sun +IMPORT_HOUR_UTC=4 +# Piso de garantia: se o ultimo import bem-sucedido for mais antigo que isto, +# um import roda na inicializacao (cobre a instancia fora do ar no horario). +IMPORT_MAX_AGE_HOURS=144 +DEFAULT_PROBE_INTERVAL_SECONDS=10800 +MIN_PROBE_INTERVAL_SECONDS=1800 +PROBE_CONCURRENCY=10 +PROBE_TIMEOUT_SECONDS=10 +MAX_RESPONSE_BYTES=1048576 +RAW_RESULT_RETENTION_DAYS=90 +ENABLE_IPV4=true +ENABLE_IPV6=auto +GEOIP_ASN_DB= + +# Fontes de lista: os padroes ficam no codigo (app/config.py). Descomente e +# defina uma linha apenas para sobrescrever a URL correspondente. +#TRACKERSLIST_ALL_URL= +#TRACKERSLIST_BEST_URL= +#NEWTRACKON_ALL_URL= +#NEWTRACKON_STABLE_URL= +#NEWTRACKON_LIVE_URL= +#JOKEPOOL710_ALL_URL= +#XIU2_ALL_URL= +#XIU2_BEST_URL=
@@ -0,0 +1,43 @@
+# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +dist/ +build/ +.venv/ +venv/ + +# Secrets — keep .env.example tracked +.env +.env.local +.env.*.local + +# SQLite / local state +data/*.db +data/*.db-* +*.sqlite +*.sqlite3 +!data/.gitkeep +backups/ + +# IDE / OS +.idea/ +.vscode/ +*.swp +.DS_Store +Thumbs.db +desktop.ini + +# Test / coverage +.coverage +.coverage.* +coverage.xml +htmlcov/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ + +# Logs +*.log
@@ -0,0 +1,43 @@
+# syntax=docker/dockerfile:1 + +FROM python:3.13-slim AS builder + +WORKDIR /build +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml README.md LICENSE ./ +COPY app ./app +COPY alembic ./alembic +COPY alembic.ini ./ + +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir . + + +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOME=/app + +WORKDIR /app + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin rastro \ + && mkdir -p /app/data \ + && chown -R rastro:rastro /app + +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin +COPY --chown=rastro:rastro app ./app +COPY --chown=rastro:rastro alembic ./alembic +COPY --chown=rastro:rastro alembic.ini pyproject.toml LICENSE README.md ./ + +USER rastro +EXPOSE 8090 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/healthz', timeout=3)" + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8090", "--workers", "1"]
@@ -0,0 +1,21 @@
+MIT License + +Copyright (c) 2026 Pablo Murad + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
@@ -0,0 +1,7 @@
+# Rastro + + + +Rastro is a self-hosted box that watches public BitTorrent trackers: it pulls the lists people already publish, knocks on each `udp`/`http`/`https` announce URL, writes down whether the peer actually speaks the protocol, and serves the history as a page, JSON, and a short curated text list. + +WE DO NOT SHARE FILES.
@@ -0,0 +1,40 @@
+[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os + +sqlalchemy.url = sqlite+aiosqlite:///data/rastro.db + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S
@@ -0,0 +1,63 @@
+from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from app.config import get_settings +from app.db.models import Base +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata +settings = get_settings() +config.set_main_option("sqlalchemy.url", settings.database_url) + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + render_as_batch=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online()
@@ -0,0 +1,21 @@
+"""Alembic revision template.""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +{{ imports if imports else "" }} + +revision: str = {{ repr(up_revision) }} +down_revision: Union[str, None] = {{ repr(down_revision) }} +branch_labels: Union[str, Sequence[str], None] = {{ repr(branch_labels) }} +depends_on: Union[str, Sequence[str], None] = {{ repr(depends_on) }} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"}
@@ -0,0 +1,209 @@
+"""Initial schema.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0001_initial" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "trackers", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("canonical_url", sa.String(length=512), nullable=False), + sa.Column("scheme", sa.String(length=16), nullable=False), + sa.Column("hostname", sa.String(length=255), nullable=False), + sa.Column("port", sa.Integer(), nullable=False), + sa.Column("path", sa.String(length=512), nullable=False), + sa.Column("current_status", sa.String(length=32), nullable=False), + sa.Column( + "first_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column( + "last_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("next_check_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("announced_interval_seconds", sa.Integer(), nullable=True), + sa.Column("last_latency_ms", sa.Float(), nullable=True), + sa.Column("supports_ipv4", sa.Boolean(), nullable=True), + sa.Column("supports_ipv6", sa.String(length=32), nullable=True), + sa.Column("terminal_cname", sa.String(length=255), nullable=True), + sa.Column("infrastructure_fingerprint", sa.String(length=128), nullable=True), + sa.Column("asn", sa.Integer(), nullable=True), + sa.Column("network_name", sa.String(length=255), nullable=True), + sa.Column("country_code", sa.String(length=8), nullable=True), + sa.Column("consecutive_failures", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("canonical_url"), + ) + op.create_index("ix_trackers_hostname", "trackers", ["hostname"]) + op.create_index("ix_trackers_current_status", "trackers", ["current_status"]) + op.create_index("ix_trackers_next_check_at", "trackers", ["next_check_at"]) + op.create_index( + "ix_trackers_infrastructure_fingerprint", "trackers", ["infrastructure_fingerprint"] + ) + + op.create_table( + "tracker_sources", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("tracker_id", sa.Integer(), nullable=False), + sa.Column("source_name", sa.String(length=64), nullable=False), + sa.Column("source_url", sa.String(length=512), nullable=False), + sa.Column("is_best_at_source", sa.Boolean(), nullable=False), + sa.Column("is_stable_at_source", sa.Boolean(), nullable=False), + sa.Column("is_live_at_source", sa.Boolean(), nullable=False), + sa.Column( + "first_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column( + "last_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.ForeignKeyConstraint(["tracker_id"], ["trackers.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tracker_id", "source_name", name="uq_tracker_source"), + ) + op.create_index("ix_tracker_sources_tracker_id", "tracker_sources", ["tracker_id"]) + + op.create_table( + "probe_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("tracker_id", sa.Integer(), nullable=False), + sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("latency_ms", sa.Float(), nullable=True), + sa.Column("ip_address", sa.String(length=64), nullable=True), + sa.Column("ip_family", sa.String(length=8), nullable=True), + sa.Column("response_valid", sa.Boolean(), nullable=False), + sa.Column("tracker_interval_seconds", sa.Integer(), nullable=True), + sa.Column("seeders", sa.Integer(), nullable=True), + sa.Column("leechers", sa.Integer(), nullable=True), + sa.Column("error_kind", sa.String(length=64), nullable=True), + sa.Column("error_detail", sa.String(length=512), nullable=True), + sa.ForeignKeyConstraint(["tracker_id"], ["trackers.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_probe_results_tracker_id", "probe_results", ["tracker_id"]) + op.create_index("ix_probe_results_checked_at", "probe_results", ["checked_at"]) + + op.create_table( + "dns_records", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("tracker_id", sa.Integer(), nullable=False), + sa.Column("record_type", sa.String(length=16), nullable=False), + sa.Column("value", sa.String(length=512), nullable=False), + sa.Column( + "first_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.Column( + "last_seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.ForeignKeyConstraint(["tracker_id"], ["trackers.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("tracker_id", "record_type", "value", name="uq_dns_record"), + ) + op.create_index("ix_dns_records_tracker_id", "dns_records", ["tracker_id"]) + + op.create_table( + "source_import_runs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("source_name", sa.String(length=64), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("received_count", sa.Integer(), nullable=False), + sa.Column("valid_count", sa.Integer(), nullable=False), + sa.Column("new_count", sa.Integer(), nullable=False), + sa.Column("unsupported_count", sa.Integer(), nullable=False), + sa.Column("error_message", sa.String(length=512), nullable=True), + sa.Column("etag", sa.String(length=255), nullable=True), + sa.Column("last_modified", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_source_import_runs_source_name", "source_import_runs", ["source_name"]) + + op.create_table( + "collector_cache", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("source_name", sa.String(length=64), nullable=False), + sa.Column("feed_url", sa.String(length=512), nullable=False), + sa.Column("etag", sa.String(length=255), nullable=True), + sa.Column("last_modified", sa.String(length=255), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("feed_url"), + ) + + op.create_table( + "app_state", + sa.Column("key", sa.String(length=64), nullable=False), + sa.Column("value", sa.Text(), nullable=False), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=False, + ), + sa.PrimaryKeyConstraint("key"), + ) + + +def downgrade() -> None: + op.drop_table("app_state") + op.drop_table("collector_cache") + op.drop_index("ix_source_import_runs_source_name", table_name="source_import_runs") + op.drop_table("source_import_runs") + op.drop_index("ix_dns_records_tracker_id", table_name="dns_records") + op.drop_table("dns_records") + op.drop_index("ix_probe_results_checked_at", table_name="probe_results") + op.drop_index("ix_probe_results_tracker_id", table_name="probe_results") + op.drop_table("probe_results") + op.drop_index("ix_tracker_sources_tracker_id", table_name="tracker_sources") + op.drop_table("tracker_sources") + op.drop_index("ix_trackers_infrastructure_fingerprint", table_name="trackers") + op.drop_index("ix_trackers_next_check_at", table_name="trackers") + op.drop_index("ix_trackers_current_status", table_name="trackers") + op.drop_index("ix_trackers_hostname", table_name="trackers") + op.drop_table("trackers")
@@ -0,0 +1,4 @@
+from app.cli import main + +if __name__ == "__main__": + raise SystemExit(main())
@@ -0,0 +1,247 @@
+from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import Settings, get_settings +from app.db import get_session +from app.db.models import ProbeResult, Tracker +from app.db.queries import get_app_state, get_tracker, overview_stats, trackers_query +from app.i18n import ( + html_lang, + lang_redirect, + render_bootstrap_note, + resolve_lang, + translate, + translator, +) +from app.services.scoring import load_metrics +from app.services.smart_list import SmartListFilters, build_smart_list, format_smart_list_txt + +router = APIRouter() +TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates" +templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) + + +def _fmt_pct(value: float | None) -> str: + if value is None: + return "—" + return f"{value * 100:.1f}%" + + +def _fmt_ms(value: float | None) -> str: + if value is None: + return "—" + return f"{value:.0f} ms" + + +def _fmt_dt(value: datetime | None) -> str: + if value is None: + return "—" + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.strftime("%Y-%m-%d %H:%M UTC") + + +templates.env.filters["pct"] = _fmt_pct +templates.env.filters["ms"] = _fmt_ms +templates.env.filters["dt"] = _fmt_dt + + +def _base_ctx(request: Request, settings: Settings, **extra: Any) -> dict[str, Any]: + lang = resolve_lang(request) + ctx = { + "request": request, + "app_name": settings.app_name, + "app_description": translate(lang, "tagline"), + "app_version": settings.app_version, + "repository_url": settings.repository_url, + "lang": lang, + "html_lang": html_lang(lang), + "t": translator(lang), + } + ctx.update(extra) + return ctx + + +@router.get("/lang/{code}") +async def set_lang(code: str, request: Request) -> RedirectResponse: + return lang_redirect(request, code) + + +@router.get("/", response_class=HTMLResponse) +async def home( + request: Request, + session: AsyncSession = Depends(get_session), + settings: Settings = Depends(get_settings), + q: str | None = None, + scheme: str | None = None, + status: str | None = None, + source: str | None = None, + sort: str = "checked", + order: str = "desc", + page: int = Query(1, ge=1), +) -> HTMLResponse: + per_page = 50 + stats = await overview_stats(session) + lang = resolve_lang(request) + bootstrap_note = render_bootstrap_note(lang, await get_app_state(session, "bootstrap_note")) + stmt = trackers_query(q=q, scheme=scheme, status=status, source=source, sort=sort, order=order) + total = int( + await session.scalar(select(func.count()).select_from(stmt.order_by(None).subquery())) or 0 + ) + rows = await session.execute(stmt.offset((page - 1) * per_page).limit(per_page)) + trackers = list(rows.scalars().unique().all()) + + table_rows = [] + for t in trackers: + metrics = await load_metrics(session, t) + alias_count = 0 + if t.infrastructure_fingerprint: + alias_count = int( + await session.scalar( + select(func.count()) + .select_from(Tracker) + .where( + Tracker.infrastructure_fingerprint == t.infrastructure_fingerprint, + Tracker.id != t.id, + ) + ) + or 0 + ) + table_rows.append( + { + "tracker": t, + "metrics": metrics, + "alias_count": alias_count, + "sources": ", ".join(sorted({s.source_name for s in t.sources})), + } + ) + + return templates.TemplateResponse( + request, + "index.html", + _base_ctx( + request, + settings, + stats=stats, + bootstrap_note=bootstrap_note, + rows=table_rows, + q=q or "", + scheme=scheme or "", + status=status or "", + source=source or "", + sort=sort, + order=order, + page=page, + total=total, + per_page=per_page, + pages=max(1, (total + per_page - 1) // per_page), + active_nav="trackers", + ), + ) + + +@router.get("/trackers/{tracker_id}", response_class=HTMLResponse) +async def tracker_page( + tracker_id: int, + request: Request, + session: AsyncSession = Depends(get_session), + settings: Settings = Depends(get_settings), +) -> HTMLResponse: + tracker = await get_tracker(session, tracker_id) + if tracker is None: + return HTMLResponse(translate(resolve_lang(request), "not_found"), status_code=404) + metrics = await load_metrics(session, tracker) + history = await session.execute( + select(ProbeResult) + .where(ProbeResult.tracker_id == tracker.id) + .order_by(ProbeResult.checked_at.desc()) + .limit(40) + ) + probes = list(history.scalars().all()) + aliases: list[Tracker] = [] + if tracker.infrastructure_fingerprint: + alias_rows = await session.execute( + select(Tracker) + .where( + Tracker.infrastructure_fingerprint == tracker.infrastructure_fingerprint, + Tracker.id != tracker.id, + ) + .limit(20) + ) + aliases = list(alias_rows.scalars().all()) + + # oldest→newest so the sparkline draws left to right + spark = [p.latency_ms or 0 for p in reversed(probes) if p.latency_ms is not None][-24:] + + return templates.TemplateResponse( + request, + "tracker_detail.html", + _base_ctx( + request, + settings, + tracker=tracker, + metrics=metrics, + probes=probes, + aliases=aliases, + spark=spark, + active_nav="trackers", + ), + ) + + +@router.get("/lists", response_class=HTMLResponse) +async def lists_page( + request: Request, + session: AsyncSession = Depends(get_session), + settings: Settings = Depends(get_settings), + protocol: str | None = None, + min_uptime: float = 0.95, + max_latency: float | None = None, + ip: str = "any", + limit: int = Query(20, ge=1, le=50), + min_age_days: float = 0.0, + diversity: bool = True, +) -> HTMLResponse: + filters = SmartListFilters( + protocol=protocol or None, + min_uptime=min_uptime, + max_latency_ms=max_latency, + ip_version=ip, + limit=limit, + min_age_days=min_age_days, + diversity=diversity, + ) + entries = await build_smart_list(session, filters) + txt = format_smart_list_txt(entries) + qs = str(request.url.query) + api_txt = f"/api/v1/lists/smart.txt?{qs}" if qs else "/api/v1/lists/smart.txt" + api_json = f"/api/v1/lists/smart.json?{qs}" if qs else "/api/v1/lists/smart.json" + return templates.TemplateResponse( + request, + "lists.html", + _base_ctx( + request, + settings, + entries=entries, + txt=txt, + protocol=protocol or "", + min_uptime=min_uptime, + max_latency=max_latency if max_latency is not None else "", + ip=ip, + limit=limit, + min_age_days=min_age_days, + diversity=diversity, + api_txt=api_txt, + api_json=api_json, + active_nav="lists", + ), + )
@@ -0,0 +1,229 @@
+from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import Settings, get_settings +from app.db import get_session +from app.db.models import Tracker +from app.db.queries import get_tracker, trackers_query +from app.services.scoring import load_metrics +from app.services.smart_list import ( + SmartListFilters, + build_live_list, + build_smart_list, + format_smart_list_json, + format_smart_list_txt, +) + +router = APIRouter() + + +def _iso(dt: datetime | None) -> str | None: + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.isoformat().replace("+00:00", "Z") + + +@router.get("/meta") +async def meta(settings: Settings = Depends(get_settings)) -> dict[str, Any]: + return { + "name": settings.app_name, + "description": settings.app_description, + "version": settings.app_version, + "public_url": settings.public_url, + "repository_url": settings.repository_url or None, + "timezone": "UTC", + "generated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + } + + +@router.get("/trackers") +async def list_trackers( + session: AsyncSession = Depends(get_session), + q: str | None = None, + scheme: str | None = None, + status: str | None = None, + source: str | None = None, + sort: str = "checked", + order: str = "desc", + page: int = Query(1, ge=1), + per_page: int = Query(50, ge=1, le=200), +) -> dict[str, Any]: + stmt = trackers_query(q=q, scheme=scheme, status=status, source=source, sort=sort, order=order) + count_stmt = select(func.count()).select_from(stmt.order_by(None).subquery()) + total = int(await session.scalar(count_stmt) or 0) + rows = await session.execute(stmt.offset((page - 1) * per_page).limit(per_page)) + trackers = list(rows.scalars().unique().all()) + items = [] + for t in trackers: + metrics = await load_metrics(session, t) + items.append(_tracker_dict(t, metrics)) + return { + "total": total, + "page": page, + "per_page": per_page, + "items": items, + } + + +@router.get("/trackers/{tracker_id}") +async def tracker_detail( + tracker_id: int, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + tracker = await get_tracker(session, tracker_id) + if tracker is None: + raise HTTPException(status_code=404, detail="not found") + metrics = await load_metrics(session, tracker) + data = _tracker_dict(tracker, metrics) + data["sources"] = [ + { + "name": s.source_name, + "is_best_at_source": s.is_best_at_source, + "is_stable_at_source": s.is_stable_at_source, + "is_live_at_source": s.is_live_at_source, + "last_seen_at": _iso(s.last_seen_at), + } + for s in tracker.sources + ] + data["dns_records"] = [ + {"type": d.record_type, "value": d.value, "last_seen_at": _iso(d.last_seen_at)} + for d in tracker.dns_records + ] + return data + + +def _tracker_dict(t: Tracker, metrics) -> dict[str, Any]: + return { + "id": t.id, + "url": t.canonical_url, + "scheme": t.scheme, + "hostname": t.hostname, + "port": t.port, + "status": t.current_status, + "score": metrics.score, + "provisional": metrics.provisional, + "uptime_24h": metrics.uptime_24h, + "uptime_7d": metrics.uptime_7d, + "uptime_30d": metrics.uptime_30d, + "latency_median_7d": metrics.latency_median_7d, + "latency_p95_7d": metrics.latency_p95_7d, + "valid_rate_7d": metrics.valid_rate_7d, + "measurement_count": metrics.measurement_count, + "supports_ipv4": t.supports_ipv4, + "supports_ipv6": t.supports_ipv6, + "asn": t.asn, + "network_name": t.network_name, + "country_code": t.country_code, + "infrastructure_fingerprint": t.infrastructure_fingerprint, + "terminal_cname": t.terminal_cname, + "last_checked_at": _iso(t.last_checked_at), + "first_seen_at": _iso(t.first_seen_at), + "sources": [s.source_name for s in (t.sources or [])], + } + + +def _etag_for(content: str | bytes) -> str: + if isinstance(content, str): + content = content.encode("utf-8") + return '"' + hashlib.sha256(content).hexdigest()[:32] + '"' + + +@router.get("/lists/smart.txt") +async def smart_list_txt( + request: Request, + session: AsyncSession = Depends(get_session), + protocol: str | None = None, + min_uptime: float = 0.95, + max_latency: float | None = None, + ip: str = "any", + limit: int = Query(20, ge=1, le=50), + min_age_days: float = 0.0, + diversity: bool = True, +) -> Response: + filters = SmartListFilters( + protocol=protocol, + min_uptime=min_uptime, + max_latency_ms=max_latency, + ip_version=ip, + limit=limit, + min_age_days=min_age_days, + diversity=diversity, + ) + entries = await build_smart_list(session, filters) + body = format_smart_list_txt(entries) + etag = _etag_for(body) + if request.headers.get("if-none-match") == etag: + return Response( + status_code=304, headers={"ETag": etag, "Cache-Control": "public, max-age=60"} + ) + return Response( + content=body, + media_type="text/plain; charset=utf-8", + headers={"ETag": etag, "Cache-Control": "public, max-age=60"}, + ) + + +@router.get("/lists/smart.json") +async def smart_list_json( + request: Request, + session: AsyncSession = Depends(get_session), + protocol: str | None = None, + min_uptime: float = 0.95, + max_latency: float | None = None, + ip: str = "any", + limit: int = Query(20, ge=1, le=50), + min_age_days: float = 0.0, + diversity: bool = True, +) -> Response: + filters = SmartListFilters( + protocol=protocol, + min_uptime=min_uptime, + max_latency_ms=max_latency, + ip_version=ip, + limit=limit, + min_age_days=min_age_days, + diversity=diversity, + ) + entries = await build_smart_list(session, filters) + payload = format_smart_list_json(entries) + import orjson + + body = orjson.dumps(payload) + etag = _etag_for(body) + if request.headers.get("if-none-match") == etag: + return Response( + status_code=304, headers={"ETag": etag, "Cache-Control": "public, max-age=60"} + ) + return Response( + content=body, + media_type="application/json", + headers={"ETag": etag, "Cache-Control": "public, max-age=60"}, + ) + + +@router.get("/lists/live.txt") +async def live_list_txt( + request: Request, + session: AsyncSession = Depends(get_session), +) -> Response: + urls = await build_live_list(session) + body = ("\n\n".join(urls) + "\n") if urls else "" + etag = _etag_for(body) + if request.headers.get("if-none-match") == etag: + return Response( + status_code=304, headers={"ETag": etag, "Cache-Control": "public, max-age=30"} + ) + return Response( + content=body, + media_type="text/plain; charset=utf-8", + headers={"ETag": etag, "Cache-Control": "public, max-age=30"}, + )
@@ -0,0 +1,73 @@
+from __future__ import annotations + +import argparse +import asyncio +import logging +import sys + +from app.collectors import import_all_sources +from app.config import get_settings +from app.db import get_engine, get_session_factory, init_db +from app.db.queries import prune_probe_results +from app.probes import probe_due_batch + + +def _setup() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + get_settings() + get_engine() + + +async def cmd_import_sources() -> int: + _setup() + await init_db() + factory = get_session_factory() + async with factory() as session: + results = await import_all_sources(session) + for name, stats in results.items(): + print(f"{name}: {stats}") + return 0 + + +async def cmd_probe_once() -> int: + _setup() + await init_db() + settings = get_settings() + factory = get_session_factory() + async with factory() as session: + count = await probe_due_batch(session, settings) + print(f"probed {count} trackers") + return 0 + + +async def cmd_prune() -> int: + _setup() + await init_db() + settings = get_settings() + factory = get_session_factory() + async with factory() as session: + removed = await prune_probe_results(session, settings.raw_result_retention_days) + print(f"removed {removed} probe results") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m app.cli") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("import-sources", help="Import tracker sources once") + sub.add_parser("probe-once", help="Probe due trackers once") + sub.add_parser("prune", help="Prune expired raw probe results") + args = parser.parse_args(argv) + + if args.command == "import-sources": + return asyncio.run(cmd_import_sources()) + if args.command == "probe-once": + return asyncio.run(cmd_probe_once()) + if args.command == "prune": + return asyncio.run(cmd_prune()) + parser.error("unknown command") + return 2 + + +if __name__ == "__main__": + sys.exit(main())
@@ -0,0 +1,52 @@
+from __future__ import annotations + +import logging + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.jokepool710 import Jokepool710Collector +from app.collectors.newtrackon import NewTrackonCollector +from app.collectors.trackerslist import TrackerslistCollector +from app.collectors.xiu2 import Xiu2Collector +from app.config import Settings, get_settings + +logger = logging.getLogger(__name__) + + +async def import_all_sources( + session: AsyncSession, + settings: Settings | None = None, + client: httpx.AsyncClient | None = None, +) -> dict[str, object]: + settings = settings or get_settings() + collectors = [ + TrackerslistCollector(), + NewTrackonCollector(), + Jokepool710Collector(), + Xiu2Collector(), + ] + owns_client = client is None + if client is None: + client = httpx.AsyncClient( + timeout=settings.probe_timeout_seconds, + headers={"User-Agent": settings.user_agent()}, + follow_redirects=True, + ) + results: dict[str, object] = {} + try: + for collector in collectors: + stats = await collector.import_once(session, client, settings) + results[collector.name] = stats + logger.info( + "import %s: received=%s valid=%s new=%s error=%s", + collector.name, + stats.received, + stats.valid, + stats.new, + stats.error, + ) + finally: + if owns_client: + await client.aclose() + return results
@@ -0,0 +1,351 @@
+from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Protocol + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import Settings +from app.db.models import CollectorCache, SourceImportRun, Tracker, TrackerSource +from app.services.normalize import ( + NormalizationError, + NormalizedTracker, + normalize_tracker_url, + parse_tracker_list_text, +) + + +@dataclass +class ImportStats: + received: int = 0 + valid: int = 0 + new: int = 0 + unsupported: int = 0 + skipped_invalid: int = 0 + not_modified: bool = False + error: str | None = None + + +@dataclass +class FeedFetchResult: + body: str | None + etag: str | None + last_modified: str | None + not_modified: bool = False + status_code: int = 0 + + +async def fetch_text_feed( + client: httpx.AsyncClient, + url: str, + *, + etag: str | None = None, + last_modified: str | None = None, +) -> FeedFetchResult: + headers: dict[str, str] = {} + if etag: + headers["If-None-Match"] = etag + if last_modified: + headers["If-Modified-Since"] = last_modified + response = await client.get(url, headers=headers) + if response.status_code == 304: + return FeedFetchResult( + body=None, + etag=etag, + last_modified=last_modified, + not_modified=True, + status_code=304, + ) + response.raise_for_status() + return FeedFetchResult( + body=response.text, + etag=response.headers.get("ETag"), + last_modified=response.headers.get("Last-Modified"), + not_modified=False, + status_code=response.status_code, + ) + + +async def get_cache(session: AsyncSession, feed_url: str) -> CollectorCache | None: + result = await session.execute( + select(CollectorCache).where(CollectorCache.feed_url == feed_url) + ) + return result.scalar_one_or_none() + + +async def upsert_cache( + session: AsyncSession, + *, + source_name: str, + feed_url: str, + etag: str | None, + last_modified: str | None, +) -> None: + row = await get_cache(session, feed_url) + now = datetime.now(UTC) + if row is None: + session.add( + CollectorCache( + source_name=source_name, + feed_url=feed_url, + etag=etag, + last_modified=last_modified, + updated_at=now, + ) + ) + else: + row.etag = etag + row.last_modified = last_modified + row.updated_at = now + + +def sanitize_error(message: str, limit: int = 512) -> str: + text = " ".join(message.split()) + return text[:limit] + + +async def upsert_tracker_from_normalized( + session: AsyncSession, + normalized: NormalizedTracker, + *, + source_name: str, + source_url: str, + is_best: bool = False, + is_stable: bool = False, + is_live: bool = False, +) -> tuple[Tracker, bool]: + """Insert or update tracker + source association. Returns (tracker, created).""" + now = datetime.now(UTC) + result = await session.execute( + select(Tracker).where(Tracker.canonical_url == normalized.canonical_url) + ) + tracker = result.scalar_one_or_none() + created = False + if tracker is None: + created = True + tracker = Tracker( + canonical_url=normalized.canonical_url, + scheme=normalized.scheme, + hostname=normalized.hostname, + port=normalized.port, + path=normalized.path, + current_status="unsupported" if normalized.unsupported else "unknown", + first_seen_at=now, + last_seen_at=now, + consecutive_failures=0, + ) + session.add(tracker) + await session.flush() + else: + tracker.last_seen_at = now + if normalized.unsupported: + tracker.current_status = "unsupported" + + src_result = await session.execute( + select(TrackerSource).where( + TrackerSource.tracker_id == tracker.id, + TrackerSource.source_name == source_name, + ) + ) + association = src_result.scalar_one_or_none() + if association is None: + session.add( + TrackerSource( + tracker_id=tracker.id, + source_name=source_name, + source_url=source_url, + is_best_at_source=is_best, + is_stable_at_source=is_stable, + is_live_at_source=is_live, + first_seen_at=now, + last_seen_at=now, + ) + ) + else: + association.last_seen_at = now + association.source_url = source_url + if is_best: + association.is_best_at_source = True + if is_stable: + association.is_stable_at_source = True + if is_live: + association.is_live_at_source = True + return tracker, created + + +@dataclass +class ParsedEntries: + valid: list[NormalizedTracker] = field(default_factory=list) + unsupported: list[NormalizedTracker] = field(default_factory=list) + invalid: int = 0 + received: int = 0 + + +def parse_and_normalize(body: str) -> ParsedEntries: + raw_urls = parse_tracker_list_text(body) + parsed = ParsedEntries(received=len(raw_urls)) + seen_canonical: set[str] = set() + for raw in raw_urls: + try: + normalized = normalize_tracker_url(raw) + except NormalizationError: + parsed.invalid += 1 + continue + if normalized.canonical_url in seen_canonical: + continue + seen_canonical.add(normalized.canonical_url) + if normalized.unsupported: + parsed.unsupported.append(normalized) + else: + parsed.valid.append(normalized) + return parsed + + +@dataclass +class EnrichmentFeed: + """A secondary feed that only sets provenance flags (best/stable/live). + + Enrichment feeds are best-effort: a failure never fails the whole import and + never touches measurement data. + """ + + url: str + is_best: bool = False + is_stable: bool = False + is_live: bool = False + + +async def import_source_feeds( + session: AsyncSession, + client: httpx.AsyncClient, + *, + source_name: str, + primary_url: str, + enrichments: Sequence[EnrichmentFeed] = (), +) -> ImportStats: + """Run the full import lifecycle for a text-feed source. + + Generalizes the pattern used by the trackerslist/newTrackon collectors: + one primary feed (cached via ETag/Last-Modified, guarded against empty + responses) plus zero or more best-effort enrichment feeds. + """ + stats = ImportStats() + run = SourceImportRun( + source_name=source_name, + started_at=datetime.now(UTC), + status="running", + received_count=0, + valid_count=0, + new_count=0, + unsupported_count=0, + ) + session.add(run) + await session.flush() + + try: + cache = await get_cache(session, primary_url) + feed = await fetch_text_feed( + client, + primary_url, + etag=cache.etag if cache else None, + last_modified=cache.last_modified if cache else None, + ) + if feed.not_modified: + stats.not_modified = True + run.status = "not_modified" + run.etag = feed.etag + run.last_modified = feed.last_modified + run.finished_at = datetime.now(UTC) + await session.commit() + return stats + + if not feed.body or not feed.body.strip(): + # Empty body must not wipe existing data. + run.status = "failed" + run.error_message = sanitize_error(f"empty response from {source_name}") + run.finished_at = datetime.now(UTC) + stats.error = run.error_message + await session.commit() + return stats + + parsed = parse_and_normalize(feed.body) + stats.received = parsed.received + stats.unsupported = len(parsed.unsupported) + stats.skipped_invalid = parsed.invalid + + for item in parsed.valid: + _, created = await upsert_tracker_from_normalized( + session, + item, + source_name=source_name, + source_url=primary_url, + ) + stats.valid += 1 + if created: + stats.new += 1 + + for enrichment in enrichments: + try: + enrich_feed = await fetch_text_feed(client, enrichment.url) + if not enrich_feed.body: + continue + for item in parse_and_normalize(enrich_feed.body).valid: + await upsert_tracker_from_normalized( + session, + item, + source_name=source_name, + source_url=enrichment.url, + is_best=enrichment.is_best, + is_stable=enrichment.is_stable, + is_live=enrichment.is_live, + ) + except Exception: # noqa: BLE001 — enrichment is best-effort + continue + + await upsert_cache( + session, + source_name=source_name, + feed_url=primary_url, + etag=feed.etag, + last_modified=feed.last_modified, + ) + run.received_count = stats.received + run.valid_count = stats.valid + run.new_count = stats.new + run.unsupported_count = stats.unsupported + run.status = "ok" + run.etag = feed.etag + run.last_modified = feed.last_modified + run.finished_at = datetime.now(UTC) + await session.commit() + return stats + except Exception as exc: # noqa: BLE001 + await session.rollback() + session.add( + SourceImportRun( + source_name=source_name, + started_at=datetime.now(UTC), + finished_at=datetime.now(UTC), + status="failed", + received_count=0, + valid_count=0, + new_count=0, + unsupported_count=0, + error_message=sanitize_error(str(exc)), + ) + ) + await session.commit() + stats.error = sanitize_error(str(exc)) + return stats + + +class Collector(Protocol): + name: str + + async def import_once( + self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings + ) -> ImportStats: ...
@@ -0,0 +1,30 @@
+"""jokepool710/Torrent-_Trackers collector. + +Independent daily public-tracker list (MIT, (c) J0KEP00L). Only the aggregate +`trackers_all.txt` feed is consumed; the per-protocol files it also publishes +are subsets of that aggregate. +""" + +from __future__ import annotations + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.base import ImportStats, import_source_feeds +from app.config import Settings + +SOURCE_NAME = "jokepool710" + + +class Jokepool710Collector: + name = SOURCE_NAME + + async def import_once( + self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings + ) -> ImportStats: + return await import_source_feeds( + session, + client, + source_name=SOURCE_NAME, + primary_url=settings.jokepool710_all_url, + )
@@ -0,0 +1,27 @@
+from __future__ import annotations + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.base import EnrichmentFeed, ImportStats, import_source_feeds +from app.config import Settings + +SOURCE_NAME = "newtrackon" + + +class NewTrackonCollector: + name = SOURCE_NAME + + async def import_once( + self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings + ) -> ImportStats: + return await import_source_feeds( + session, + client, + source_name=SOURCE_NAME, + primary_url=settings.newtrackon_all_url, + enrichments=[ + EnrichmentFeed(url=settings.newtrackon_stable_url, is_stable=True), + EnrichmentFeed(url=settings.newtrackon_live_url, is_live=True), + ], + )
@@ -0,0 +1,24 @@
+from __future__ import annotations + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.base import EnrichmentFeed, ImportStats, import_source_feeds +from app.config import Settings + +SOURCE_NAME = "trackerslist" + + +class TrackerslistCollector: + name = SOURCE_NAME + + async def import_once( + self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings + ) -> ImportStats: + return await import_source_feeds( + session, + client, + source_name=SOURCE_NAME, + primary_url=settings.trackerslist_all_url, + enrichments=[EnrichmentFeed(url=settings.trackerslist_best_url, is_best=True)], + )
@@ -0,0 +1,31 @@
+"""XIU2/TrackersListCollection collector. + +Daily aggregator of several upstream tracker lists (GPL-3.0). As with the +GPLv2 trackerslist feed, only published data is consumed -- no GPL code is +used. `all.txt` seeds endpoints; `best.txt` only sets the best-at-source flag. +""" + +from __future__ import annotations + +import httpx +from sqlalchemy.ext.asyncio import AsyncSession + +from app.collectors.base import EnrichmentFeed, ImportStats, import_source_feeds +from app.config import Settings + +SOURCE_NAME = "xiu2" + + +class Xiu2Collector: + name = SOURCE_NAME + + async def import_once( + self, session: AsyncSession, client: httpx.AsyncClient, settings: Settings + ) -> ImportStats: + return await import_source_feeds( + session, + client, + source_name=SOURCE_NAME, + primary_url=settings.xiu2_all_url, + enrichments=[EnrichmentFeed(url=settings.xiu2_best_url, is_best=True)], + )
@@ -0,0 +1,136 @@
+from __future__ import annotations + +from functools import lru_cache +from typing import Literal + +from pydantic import Field, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + populate_by_name=True, + ) + + app_name: str = Field(default="Rastro", alias="APP_NAME") + app_description: str = Field( + default="Observatorio publico de trackers BitTorrent", + alias="APP_DESCRIPTION", + ) + app_version: str = Field(default="0.1.0", alias="APP_VERSION") + public_url: str = Field(default="http://localhost:8090", alias="PUBLIC_URL") + repository_url: str = Field(default="", alias="REPOSITORY_URL") + + database_url: str = Field( + default="sqlite+aiosqlite:///data/rastro.db", + alias="DATABASE_URL", + ) + host: str = Field(default="0.0.0.0", alias="HOST") + port: int = Field(default=8090, alias="PORT", ge=1, le=65535) + log_level: str = Field(default="INFO", alias="LOG_LEVEL") + + run_scheduler: bool = Field(default=True, alias="RUN_SCHEDULER") + # Source lists refresh on a weekly cron (UTC). A startup import still runs + # when the inventory is empty or the last successful import is stale. + import_day_of_week: str = Field(default="sun", alias="IMPORT_DAY_OF_WEEK") + import_hour_utc: int = Field(default=4, alias="IMPORT_HOUR_UTC", ge=0, le=23) + import_max_age_hours: int = Field(default=144, alias="IMPORT_MAX_AGE_HOURS", ge=1) + default_probe_interval_seconds: int = Field( + default=10800, alias="DEFAULT_PROBE_INTERVAL_SECONDS", ge=60 + ) + min_probe_interval_seconds: int = Field(default=1800, alias="MIN_PROBE_INTERVAL_SECONDS", ge=60) + probe_concurrency: int = Field(default=10, alias="PROBE_CONCURRENCY", ge=1, le=100) + probe_timeout_seconds: float = Field(default=10.0, alias="PROBE_TIMEOUT_SECONDS", gt=0, le=120) + max_response_bytes: int = Field(default=1_048_576, alias="MAX_RESPONSE_BYTES", ge=1024) + raw_result_retention_days: int = Field(default=90, alias="RAW_RESULT_RETENTION_DAYS", ge=1) + + enable_ipv4: bool = Field(default=True, alias="ENABLE_IPV4") + enable_ipv6: Literal["true", "false", "auto"] | bool = Field( + default="auto", alias="ENABLE_IPV6" + ) + geoip_asn_db: str = Field(default="", alias="GEOIP_ASN_DB") + + # Source URLs (overridable for tests) + trackerslist_all_url: str = Field( + default=("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt"), + alias="TRACKERSLIST_ALL_URL", + ) + trackerslist_best_url: str = Field( + default=("https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt"), + alias="TRACKERSLIST_BEST_URL", + ) + newtrackon_all_url: str = Field( + default="https://newtrackon.com/api/all", + alias="NEWTRACKON_ALL_URL", + ) + newtrackon_stable_url: str = Field( + default="https://newtrackon.com/api/stable", + alias="NEWTRACKON_STABLE_URL", + ) + newtrackon_live_url: str = Field( + default="https://newtrackon.com/api/live", + alias="NEWTRACKON_LIVE_URL", + ) + jokepool710_all_url: str = Field( + default=( + "https://raw.githubusercontent.com/jokepool710/Torrent-_Trackers/main/trackers_all.txt" + ), + alias="JOKEPOOL710_ALL_URL", + ) + xiu2_all_url: str = Field( + default="https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt", + alias="XIU2_ALL_URL", + ) + xiu2_best_url: str = Field( + default="https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt", + alias="XIU2_BEST_URL", + ) + + @field_validator("enable_ipv6", mode="before") + @classmethod + def normalize_ipv6(cls, value: object) -> object: + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "yes", "true"}: + return True + if lowered in {"0", "no", "false"}: + return False + if lowered == "auto": + return "auto" + return value + + @model_validator(mode="after") + def validate_intervals(self) -> Settings: + if self.min_probe_interval_seconds > self.default_probe_interval_seconds: + raise ValueError( + "MIN_PROBE_INTERVAL_SECONDS cannot exceed DEFAULT_PROBE_INTERVAL_SECONDS" + ) + return self + + def user_agent(self) -> str: + base = f"{self.app_name}/{self.app_version}" + url = (self.public_url or "").strip() + if url: + return f"{base} (+{url})" + return base + + def peer_id_prefix(self) -> bytes: + """Return a 8-byte BitTorrent peer_id style prefix derived from app name.""" + slug = "".join(c for c in self.app_name.upper() if c.isalnum())[:6] or "RASTRO" + slug = slug.ljust(6, "X")[:6] + return f"-{slug}-".encode("ascii") + + def ipv6_mode(self) -> Literal["true", "false", "auto"]: + if self.enable_ipv6 is True: + return "true" + if self.enable_ipv6 is False: + return "false" + return "auto" + + +@lru_cache +def get_settings() -> Settings: + return Settings()
@@ -0,0 +1,80 @@
+from __future__ import annotations + +from collections.abc import AsyncGenerator +from pathlib import Path + +from sqlalchemy import event, text +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from app.config import Settings, get_settings + +_engine: AsyncEngine | None = None +_session_factory: async_sessionmaker[AsyncSession] | None = None + + +def _ensure_sqlite_parent(database_url: str) -> None: + if "sqlite" not in database_url: + return + # sqlite+aiosqlite:///data/rastro.db or ////absolute + raw = database_url.split("///", 1)[-1] + path = Path(raw) + if path.parent and str(path.parent) not in {".", ""}: + path.parent.mkdir(parents=True, exist_ok=True) + + +def get_engine(settings: Settings | None = None) -> AsyncEngine: + global _engine, _session_factory + if _engine is not None: + return _engine + settings = settings or get_settings() + _ensure_sqlite_parent(settings.database_url) + _engine = create_async_engine( + settings.database_url, + echo=False, + connect_args={"check_same_thread": False}, + ) + + @event.listens_for(_engine.sync_engine, "connect") + def _set_sqlite_pragma(dbapi_connection, connection_record) -> None: # noqa: ARG001 + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + _session_factory = async_sessionmaker(_engine, expire_on_commit=False) + return _engine + + +def get_session_factory() -> async_sessionmaker[AsyncSession]: + if _session_factory is None: + get_engine() + assert _session_factory is not None + return _session_factory + + +async def get_session() -> AsyncGenerator[AsyncSession]: + factory = get_session_factory() + async with factory() as session: + yield session + + +async def init_db(settings: Settings | None = None) -> None: + """Create tables if needed (Alembic preferred; used as fallback/tests).""" + from app.db.models import Base + + engine = get_engine(settings) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + await conn.execute(text("PRAGMA journal_mode=WAL")) + + +def reset_engine() -> None: + """Reset global engine (tests).""" + global _engine, _session_factory + _engine = None + _session_factory = None
@@ -0,0 +1,185 @@
+from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import ( + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + func, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +class Base(DeclarativeBase): + pass + + +class Tracker(Base): + __tablename__ = "trackers" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + canonical_url: Mapped[str] = mapped_column(String(512), unique=True, nullable=False) + scheme: Mapped[str] = mapped_column(String(16), nullable=False) + hostname: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + port: Mapped[int] = mapped_column(Integer, nullable=False) + path: Mapped[str] = mapped_column(String(512), nullable=False, default="/announce") + + current_status: Mapped[str] = mapped_column( + String(32), nullable=False, default="unknown", index=True + ) + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + next_check_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, index=True + ) + announced_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + last_latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + + supports_ipv4: Mapped[bool | None] = mapped_column(Boolean, nullable=True) + supports_ipv6: Mapped[str | None] = mapped_column(String(32), nullable=True) + # supports_ipv6 stores: true / false / not_tested / unknown as string for flexibility + + terminal_cname: Mapped[str | None] = mapped_column(String(255), nullable=True) + infrastructure_fingerprint: Mapped[str | None] = mapped_column( + String(128), nullable=True, index=True + ) + asn: Mapped[int | None] = mapped_column(Integer, nullable=True) + network_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + country_code: Mapped[str | None] = mapped_column(String(8), nullable=True) + + consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + sources: Mapped[list[TrackerSource]] = relationship(back_populates="tracker") + probe_results: Mapped[list[ProbeResult]] = relationship(back_populates="tracker") + dns_records: Mapped[list[DnsRecord]] = relationship(back_populates="tracker") + + +class TrackerSource(Base): + __tablename__ = "tracker_sources" + __table_args__ = (UniqueConstraint("tracker_id", "source_name", name="uq_tracker_source"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tracker_id: Mapped[int] = mapped_column( + ForeignKey("trackers.id", ondelete="CASCADE"), nullable=False, index=True + ) + source_name: Mapped[str] = mapped_column(String(64), nullable=False) + source_url: Mapped[str] = mapped_column(String(512), nullable=False) + is_best_at_source: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_stable_at_source: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_live_at_source: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + tracker: Mapped[Tracker] = relationship(back_populates="sources") + + +class ProbeResult(Base): + __tablename__ = "probe_results" + __table_args__ = (Index("ix_probe_results_checked_at", "checked_at"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tracker_id: Mapped[int] = mapped_column( + ForeignKey("trackers.id", ondelete="CASCADE"), nullable=False, index=True + ) + checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False) + latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True) + ip_address: Mapped[str | None] = mapped_column(String(64), nullable=True) + ip_family: Mapped[str | None] = mapped_column(String(8), nullable=True) + response_valid: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + tracker_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + seeders: Mapped[int | None] = mapped_column(Integer, nullable=True) + leechers: Mapped[int | None] = mapped_column(Integer, nullable=True) + error_kind: Mapped[str | None] = mapped_column(String(64), nullable=True) + error_detail: Mapped[str | None] = mapped_column(String(512), nullable=True) + + tracker: Mapped[Tracker] = relationship(back_populates="probe_results") + + +class DnsRecord(Base): + __tablename__ = "dns_records" + __table_args__ = (UniqueConstraint("tracker_id", "record_type", "value", name="uq_dns_record"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + tracker_id: Mapped[int] = mapped_column( + ForeignKey("trackers.id", ondelete="CASCADE"), nullable=False, index=True + ) + record_type: Mapped[str] = mapped_column(String(16), nullable=False) + value: Mapped[str] = mapped_column(String(512), nullable=False) + first_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + last_seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + tracker: Mapped[Tracker] = relationship(back_populates="dns_records") + + +class SourceImportRun(Base): + __tablename__ = "source_import_runs" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + source_name: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="running") + received_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + valid_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + new_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + unsupported_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_message: Mapped[str | None] = mapped_column(String(512), nullable=True) + etag: Mapped[str | None] = mapped_column(String(255), nullable=True) + last_modified: Mapped[str | None] = mapped_column(String(255), nullable=True) + + +class CollectorCache(Base): + """HTTP cache state per source feed URL.""" + + __tablename__ = "collector_cache" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + source_name: Mapped[str] = mapped_column(String(64), nullable=False) + feed_url: Mapped[str] = mapped_column(String(512), nullable=False, unique=True) + etag: Mapped[str | None] = mapped_column(String(255), nullable=True) + last_modified: Mapped[str | None] = mapped_column(String(255), nullable=True) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + +class AppState(Base): + """Key/value operational state (bootstrap notes, etc.).""" + + __tablename__ = "app_state" + + key: Mapped[str] = mapped_column(String(64), primary_key=True) + value: Mapped[str] = mapped_column(Text, nullable=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + )
@@ -0,0 +1,151 @@
+from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import Select, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.db.models import ( + AppState, + ProbeResult, + SourceImportRun, + Tracker, + TrackerSource, +) + + +async def get_app_state(session: AsyncSession, key: str) -> str | None: + row = await session.get(AppState, key) + return row.value if row else None + + +async def set_app_state(session: AsyncSession, key: str, value: str) -> None: + row = await session.get(AppState, key) + if row is None: + session.add(AppState(key=key, value=value, updated_at=datetime.now(UTC))) + else: + row.value = value + row.updated_at = datetime.now(UTC) + await session.commit() + + +async def count_trackers(session: AsyncSession) -> int: + result = await session.scalar(select(func.count()).select_from(Tracker)) + return int(result or 0) + + +async def latest_successful_import_at(session: AsyncSession) -> datetime | None: + """Finished-at of the most recent import that actually applied data. + + `not_modified` counts as success: the upstream feed was reachable and simply + unchanged, so the inventory is not stale. + """ + return await session.scalar( + select(func.max(SourceImportRun.finished_at)).where( + SourceImportRun.status.in_(("ok", "not_modified")) + ) + ) + + +async def overview_stats(session: AsyncSession) -> dict[str, Any]: + total = await count_trackers(session) + by_status: dict[str, int] = {} + rows = await session.execute( + select(Tracker.current_status, func.count()).group_by(Tracker.current_status) + ) + for status, count in rows.all(): + by_status[status] = count + + last_import = await session.scalar( + select(SourceImportRun) + .where(SourceImportRun.status.in_(("ok", "not_modified", "failed"))) + .order_by(SourceImportRun.finished_at.desc()) + .limit(1) + ) + last_probe = await session.scalar( + select(ProbeResult).order_by(ProbeResult.checked_at.desc()).limit(1) + ) + + return { + "total": total, + "up": by_status.get("up", 0), + "degraded": by_status.get("degraded", 0), + "down": by_status.get("down", 0), + "unknown": by_status.get("unknown", 0), + "last_import_at": last_import.finished_at if last_import else None, + "last_probe_at": last_probe.checked_at if last_probe else None, + } + + +def trackers_query( + *, + q: str | None = None, + scheme: str | None = None, + status: str | None = None, + source: str | None = None, + sort: str = "score", + order: str = "desc", +) -> Select[tuple[Tracker]]: + stmt = select(Tracker).options(selectinload(Tracker.sources)) + if q: + like = f"%{q.strip()}%" + stmt = stmt.where(Tracker.canonical_url.ilike(like) | Tracker.hostname.ilike(like)) + if scheme: + stmt = stmt.where(Tracker.scheme == scheme.lower()) + if status: + stmt = stmt.where(Tracker.current_status == status.lower()) + if source: + stmt = stmt.join(TrackerSource).where(TrackerSource.source_name == source) + + sort_map = { + "url": Tracker.canonical_url, + "status": Tracker.current_status, + "latency": Tracker.last_latency_ms, + "checked": Tracker.last_checked_at, + "hostname": Tracker.hostname, + "protocol": Tracker.scheme, + } + col = sort_map.get(sort, Tracker.last_checked_at) + if order.lower() == "asc": + stmt = stmt.order_by(col.asc().nullslast(), Tracker.id.asc()) + else: + stmt = stmt.order_by(col.desc().nullslast(), Tracker.id.desc()) + return stmt + + +async def get_tracker(session: AsyncSession, tracker_id: int) -> Tracker | None: + result = await session.execute( + select(Tracker) + .options( + selectinload(Tracker.sources), + selectinload(Tracker.dns_records), + ) + .where(Tracker.id == tracker_id) + ) + return result.scalar_one_or_none() + + +async def due_trackers(session: AsyncSession, limit: int = 50) -> list[Tracker]: + now = datetime.now(UTC) + result = await session.execute( + select(Tracker) + .where( + Tracker.current_status != "unsupported", + (Tracker.next_check_at.is_(None)) | (Tracker.next_check_at <= now), + ) + .order_by(Tracker.next_check_at.asc().nullsfirst()) + .limit(limit) + ) + return list(result.scalars().all()) + + +async def prune_probe_results(session: AsyncSession, retention_days: int) -> int: + cutoff = datetime.now(UTC) - timedelta(days=retention_days) + rows = await session.execute(select(ProbeResult).where(ProbeResult.checked_at < cutoff)) + old = list(rows.scalars().all()) + for row in old: + await session.delete(row) + await session.commit() + return len(old)
@@ -0,0 +1,353 @@
+from __future__ import annotations + +from urllib.parse import urlparse + +from starlette.requests import Request +from starlette.responses import RedirectResponse + +LOCALES = ("pt", "en") +DEFAULT = "pt" +COOKIE = "lang" +COOKIE_MAX_AGE = 60 * 60 * 24 * 365 + +_PT = { + "tagline": "Observatório público de trackers BitTorrent", + "nav_trackers": "Trackers", + "nav_trackers_sub": "lista", + "nav_lists": "Criar lista", + "nav_lists_sub": "smart", + "nav_github": "GitHub", + "nav_github_sub": "código", + "footer_utc": "medições em UTC", + "footer_disclaimer": "Não é um tracker BitTorrent · não indexa torrents", + "stats_label": "Resumo", + "stats_monitored": "monitorados", + "stats_up": "ativos", + "stats_degraded": "degradados", + "stats_down": "indisponíveis", + "stats_unknown": "desconhecidos", + "stats_last_import": "última importação", + "stats_last_probe": "última medição", + "filter_search": "Busca", + "filter_search_ph": "hostname ou URL", + "filter_protocol": "Protocolo", + "filter_any": "qualquer", + "filter_status": "Status", + "filter_source": "Origem", + "filter_sort": "Ordenar", + "sort_checked": "última verificação", + "sort_latency": "latência", + "sort_url": "URL", + "sort_status": "status", + "sort_protocol": "protocolo", + "filter_order": "Ordem", + "order_desc": "desc", + "order_asc": "asc", + "filter_submit": "Filtrar", + "col_tracker": "Tracker", + "col_protocol": "Protocolo", + "col_status": "Status", + "col_uptime7": "Uptime 7d", + "col_latency": "Latência med.", + "col_ip": "IPv4 / IPv6", + "col_asn": "ASN / rede", + "col_sources": "Fontes", + "col_aliases": "Aliases", + "col_last_check": "Última verificação", + "copy": "copiar", + "copy_btn": "Copiar", + "copied": "copiado", + "th_latency": "Latência", + "provisional": "provisório", + "empty_inventory": "Nenhum tracker no inventário ainda.", + "pagination": "Paginação", + "prev": "Anterior", + "next": "Próxima", + "page_of": "Página {page} / {pages} ({total})", + "lists_title": "Criar lista", + "lists_heading": "Smart List", + "lists_blurb": ( + "Seleção de trackers estáveis e com diversidade de infraestrutura. " + "Classificações das fontes externas não substituem as medições próprias." + ), + "min_uptime": "Uptime mínimo (0–1)", + "max_latency": "Latência máxima (ms)", + "filter_ip": "IP", + "filter_limit": "Quantidade (1–50)", + "min_age": "Idade mínima (dias)", + "diversity": "Diversidade de infraestrutura", + "diversity_on": "ligada", + "diversity_off": "desligada", + "generate": "Gerar", + "selected_count": "trackers selecionados.", + "download_txt": "Baixar TXT", + "open_json": "Abrir JSON", + "permalink": "URL permanente", + "back": "voltar", + "score": "pontuação", + "score_insufficient": "insuficiente", + "uptime_24h": "Uptime 24h", + "uptime_7d": "Uptime 7d", + "uptime_30d": "Uptime 30d", + "latency_med_7d": "Latência med. 7d", + "latency_p95": "Latência p95 7d", + "valid_rate": "Respostas válidas 7d", + "measurements": "Medições", + "tracking": "Acompanhamento", + "days": "dias", + "spark_label": "Latências recentes", + "sources": "Origens", + "best_at_source": "best na origem", + "stable_at_source": "stable na origem", + "live_at_source": "live na origem", + "seen": "visto", + "no_source": "Sem associação de origem", + "dns": "DNS", + "terminal_cname": "CNAME terminal", + "infra_fp": "Fingerprint de infraestrutura", + "asn": "ASN", + "unknown": "desconhecido", + "country": "País", + "no_dns": "Sem registros DNS armazenados", + "shared_infra": "Infraestrutura compartilhada", + "shared_infra_blurb": ( + "Outros endpoints com o mesmo fingerprint (sinal, não prova de operador):" + ), + "recent_history": "Histórico recente", + "when": "Quando (UTC)", + "valid": "Válida", + "error": "Erro", + "yes": "sim", + "no": "não", + "no_probes": "Sem medições ainda", + "not_found": "Tracker não encontrado", + "bootstrap_empty": ( + "Inventário vazio: não foi possível importar fontes na inicialização ({exc}). " + "O site está no ar; tente novamente com `python -m app.cli import-sources`." + ), + "bootstrap_stale": ( + "Listas desatualizadas: não foi possível importar fontes na inicialização ({exc}). " + "O site está no ar; tente novamente com `python -m app.cli import-sources`." + ), +} + +_EN = { + "tagline": "Public observatory of BitTorrent trackers", + "nav_trackers": "Trackers", + "nav_trackers_sub": "list", + "nav_lists": "Build a list", + "nav_lists_sub": "smart", + "nav_github": "GitHub", + "nav_github_sub": "code", + "footer_utc": "measurements in UTC", + "footer_disclaimer": "Not a BitTorrent tracker · does not index torrents", + "stats_label": "Overview", + "stats_monitored": "watched", + "stats_up": "up", + "stats_degraded": "degraded", + "stats_down": "down", + "stats_unknown": "unknown", + "stats_last_import": "last import", + "stats_last_probe": "last probe", + "filter_search": "Search", + "filter_search_ph": "hostname or URL", + "filter_protocol": "Protocol", + "filter_any": "any", + "filter_status": "Status", + "filter_source": "Source", + "filter_sort": "Sort", + "sort_checked": "last check", + "sort_latency": "latency", + "sort_url": "URL", + "sort_status": "status", + "sort_protocol": "protocol", + "filter_order": "Order", + "order_desc": "desc", + "order_asc": "asc", + "filter_submit": "Filter", + "col_tracker": "Tracker", + "col_protocol": "Protocol", + "col_status": "Status", + "col_uptime7": "Uptime 7d", + "col_latency": "Median latency", + "col_ip": "IPv4 / IPv6", + "col_asn": "ASN / net", + "col_sources": "Sources", + "col_aliases": "Aliases", + "col_last_check": "Last check", + "copy": "copy", + "copy_btn": "Copy", + "copied": "copied", + "th_latency": "Latency", + "provisional": "provisional", + "empty_inventory": "No trackers in the inventory yet.", + "pagination": "Pagination", + "prev": "Previous", + "next": "Next", + "page_of": "Page {page} / {pages} ({total})", + "lists_title": "Build a list", + "lists_heading": "Smart List", + "lists_blurb": ( + "Stable trackers with some infrastructure spread. " + "Labels from the upstream lists do not replace our own probes." + ), + "min_uptime": "Min uptime (0–1)", + "max_latency": "Max latency (ms)", + "filter_ip": "IP", + "filter_limit": "Count (1–50)", + "min_age": "Min age (days)", + "diversity": "Infrastructure diversity", + "diversity_on": "on", + "diversity_off": "off", + "generate": "Build", + "selected_count": "trackers selected.", + "download_txt": "Download TXT", + "open_json": "Open JSON", + "permalink": "Permalink", + "back": "back", + "score": "score", + "score_insufficient": "not enough data", + "uptime_24h": "Uptime 24h", + "uptime_7d": "Uptime 7d", + "uptime_30d": "Uptime 30d", + "latency_med_7d": "Median latency 7d", + "latency_p95": "p95 latency 7d", + "valid_rate": "Valid replies 7d", + "measurements": "Probes", + "tracking": "Watched for", + "days": "days", + "spark_label": "Recent latencies", + "sources": "Sources", + "best_at_source": "best at source", + "stable_at_source": "stable at source", + "live_at_source": "live at source", + "seen": "seen", + "no_source": "No source association", + "dns": "DNS", + "terminal_cname": "Terminal CNAME", + "infra_fp": "Infrastructure fingerprint", + "asn": "ASN", + "unknown": "unknown", + "country": "Country", + "no_dns": "No stored DNS records", + "shared_infra": "Shared infrastructure", + "shared_infra_blurb": ( + "Other endpoints with the same fingerprint (a hint, not proof of the same operator):" + ), + "recent_history": "Recent history", + "when": "When (UTC)", + "valid": "Valid", + "error": "Error", + "yes": "yes", + "no": "no", + "no_probes": "No probes yet", + "not_found": "Tracker not found", + "bootstrap_empty": ( + "Empty inventory: could not import sources on startup ({exc}). " + "The site is up; retry with `python -m app.cli import-sources`." + ), + "bootstrap_stale": ( + "Stale lists: could not import sources on startup ({exc}). " + "The site is up; retry with `python -m app.cli import-sources`." + ), +} + +STRINGS = {"pt": _PT, "en": _EN} + +_BOOTSTRAP_KEYS = { + "empty_import": "bootstrap_empty", + "stale_import": "bootstrap_stale", +} + + +def normalize_lang(code: str | None) -> str: + if not code: + return DEFAULT + tag = code.strip().lower().replace("_", "-") + if tag in LOCALES: + return tag + if tag.startswith("en"): + return "en" + if tag.startswith("pt"): + return "pt" + return DEFAULT + + +def from_accept_language(header: str | None) -> str: + if not header: + return DEFAULT + for part in header.split(","): + tag = part.split(";", 1)[0].strip() + if not tag: + continue + return normalize_lang(tag) + return DEFAULT + + +def resolve_lang(request: Request) -> str: + cookie = request.cookies.get(COOKIE) + if cookie in LOCALES: + return cookie + return from_accept_language(request.headers.get("accept-language")) + + +def html_lang(lang: str) -> str: + return "pt-BR" if lang == "pt" else "en" + + +def translate(lang: str, key: str, **kwargs: object) -> str: + table = STRINGS.get(lang) or STRINGS[DEFAULT] + text = table.get(key) or STRINGS[DEFAULT].get(key) or key + if kwargs: + return text.format(**kwargs) + return text + + +def translator(lang: str): + def _t(key: str, **kwargs: object) -> str: + return translate(lang, key, **kwargs) + + return _t + + +def encode_bootstrap_note(kind: str, exc: object) -> str: + return f"{kind}|{exc}" + + +def render_bootstrap_note(lang: str, raw: str | None) -> str: + if not raw: + return "" + kind, sep, detail = raw.partition("|") + key = _BOOTSTRAP_KEYS.get(kind) + if key and sep: + return translate(lang, key, exc=detail) + return raw + + +def safe_back(request: Request) -> str: + ref = request.headers.get("referer") + if not ref: + return "/" + parsed = urlparse(ref) + if parsed.netloc and parsed.netloc != request.url.netloc: + return "/" + path = parsed.path or "/" + if path.startswith("/lang"): + return "/" + if parsed.query: + return f"{path}?{parsed.query}" + return path + + +def lang_redirect(request: Request, code: str) -> RedirectResponse: + lang = normalize_lang(code) + resp = RedirectResponse(safe_back(request), status_code=303) + resp.set_cookie( + COOKIE, + lang, + max_age=COOKIE_MAX_AGE, + path="/", + samesite="lax", + httponly=True, + ) + return resp
@@ -0,0 +1,128 @@
+from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from fastapi import FastAPI +from fastapi.responses import PlainTextResponse +from fastapi.staticfiles import StaticFiles +from sqlalchemy import text + +from app.api import pages, routes +from app.collectors import import_all_sources +from app.config import get_settings +from app.db import get_engine, get_session_factory, init_db +from app.db.queries import count_trackers, latest_successful_import_at, set_app_state +from app.i18n import encode_bootstrap_note +from app.scheduler import start_scheduler, stop_scheduler + +logger = logging.getLogger(__name__) + + +def _configure_logging(level: str) -> None: + # force=True: uvicorn configures logging before lifespan runs, so without it + # basicConfig is a no-op and app logs (scheduler, imports) never reach stdout. + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s [%(name)s] %(message)s", + force=True, + ) + + +async def _run_migrations() -> None: + """Apply Alembic migrations; fall back to create_all if needed.""" + try: + from alembic import command + from alembic.config import Config + + cfg = Config(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + settings = get_settings() + cfg.set_main_option("sqlalchemy.url", settings.database_url) + await __import__("asyncio").to_thread(command.upgrade, cfg, "head") + except Exception: # noqa: BLE001 + logger.exception("Alembic upgrade failed; falling back to create_all") + await init_db() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + _configure_logging(settings.log_level) + get_engine(settings) + await _run_migrations() + + factory = get_session_factory() + async with factory() as session: + total = await count_trackers(session) + last_import = await latest_successful_import_at(session) + max_age = timedelta(hours=settings.import_max_age_hours) + if last_import is not None and last_import.tzinfo is None: + last_import = last_import.replace(tzinfo=UTC) + stale = last_import is None or (datetime.now(UTC) - last_import) > max_age + if total == 0 or stale: + try: + await import_all_sources(session, settings) + await set_app_state(session, "bootstrap_note", "") + except Exception as exc: # noqa: BLE001 + kind = "empty_import" if total == 0 else "stale_import" + logger.warning("bootstrap import failed (%s): %s", kind, exc) + await set_app_state(session, "bootstrap_note", encode_bootstrap_note(kind, exc)) + + if settings.run_scheduler: + start_scheduler(settings) + yield + stop_scheduler() + + +def create_app() -> FastAPI: + settings = get_settings() + app = FastAPI( + title=settings.app_name, + description=settings.app_description, + version=settings.app_version, + lifespan=lifespan, + ) + + static_dir = Path(__file__).resolve().parent / "static" + app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") + + app.include_router(pages.router) + app.include_router(routes.router, prefix="/api/v1") + + @app.get("/healthz", response_class=PlainTextResponse) + async def healthz() -> str: + return "ok" + + @app.get("/readyz", response_class=PlainTextResponse) + async def readyz() -> PlainTextResponse: + try: + engine = get_engine() + async with engine.connect() as conn: + await conn.execute(text("SELECT 1")) + return PlainTextResponse("ready") + except Exception: # noqa: BLE001 + return PlainTextResponse("not ready", status_code=503) + + return app + + +app = create_app() + + +def run() -> None: + import uvicorn + + settings = get_settings() + uvicorn.run( + "app.main:app", + host=settings.host, + port=settings.port, + workers=1, + log_level=settings.log_level.lower(), + ) + + +if __name__ == "__main__": + run()
@@ -0,0 +1,209 @@
+from __future__ import annotations + +import asyncio +import logging +import random +from datetime import UTC, datetime, timedelta + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import Settings +from app.db.models import DnsRecord, ProbeResult, Tracker +from app.probes.http import probe_http +from app.probes.udp import probe_udp +from app.services.dns import host_has_ipv6, lookup_asn, resolve_cname_chain + +logger = logging.getLogger(__name__) + + +def compute_next_check( + *, + settings: Settings, + announced_interval: int | None, + consecutive_failures: int, + success: bool, +) -> datetime: + now = datetime.now(UTC) + if success: + base = announced_interval or settings.default_probe_interval_seconds + base = max(base, settings.min_probe_interval_seconds) + else: + exp = min(consecutive_failures, 12) + base = min( + settings.min_probe_interval_seconds * (2 ** max(exp - 1, 0)), + 12 * 3600, + ) + base = max(base, settings.min_probe_interval_seconds) + jitter = random.uniform(0, min(60.0, base * 0.1)) + return now + timedelta(seconds=base + jitter) + + +async def _upsert_dns_records( + session: AsyncSession, tracker: Tracker, records: list[tuple[str, str]] +) -> None: + now = datetime.now(UTC) + for rtype, value in records: + result = await session.execute( + select(DnsRecord).where( + DnsRecord.tracker_id == tracker.id, + DnsRecord.record_type == rtype, + DnsRecord.value == value, + ) + ) + row = result.scalar_one_or_none() + if row is None: + session.add( + DnsRecord( + tracker_id=tracker.id, + record_type=rtype, + value=value, + first_seen_at=now, + last_seen_at=now, + ) + ) + else: + row.last_seen_at = now + + +async def probe_tracker( + session: AsyncSession, + tracker: Tracker, + settings: Settings, + client: httpx.AsyncClient, +) -> ProbeResult: + now = datetime.now(UTC) + dns = await resolve_cname_chain(tracker.hostname) + tracker.terminal_cname = dns.terminal_cname + tracker.infrastructure_fingerprint = dns.fingerprint + + dns_rows: list[tuple[str, str]] = [("CNAME", c) for c in dns.cname_chain] + dns_rows.extend(("A", ip) for ip in dns.ipv4) + dns_rows.extend(("AAAA", ip) for ip in dns.ipv6) + await _upsert_dns_records(session, tracker, dns_rows) + + ipv6_mode = settings.ipv6_mode() + can_v6 = host_has_ipv6() if ipv6_mode == "auto" else ipv6_mode == "true" + tracker.supports_ipv6 = "not_tested" if not can_v6 else ("true" if dns.ipv6 else "false") + tracker.supports_ipv4 = bool(dns.ipv4) if settings.enable_ipv4 else None + + public_ips = list(dns.ipv4) + if can_v6: + public_ips.extend(dns.ipv6) + + if settings.geoip_asn_db and public_ips: + asn, net, country = lookup_asn(public_ips[0], settings.geoip_asn_db) + tracker.asn = asn + tracker.network_name = net + tracker.country_code = country + + if tracker.scheme == "udp": + peer = (settings.peer_id_prefix() + b"xxxxxxxxxxxx")[:20] + outcome = await probe_udp( + tracker.hostname, + tracker.port, + timeout=settings.probe_timeout_seconds, + peer_id=peer, + ) + elif tracker.scheme in {"http", "https"}: + outcome = await probe_http( + tracker.canonical_url, + client=client, + timeout=settings.probe_timeout_seconds, + max_bytes=settings.max_response_bytes, + peer_id_prefix=settings.peer_id_prefix(), + resolved_ips=public_ips or None, + ) + else: + tracker.current_status = "unsupported" + tracker.last_checked_at = now + tracker.next_check_at = now + timedelta(days=30) + result = ProbeResult( + tracker_id=tracker.id, + checked_at=now, + status="unsupported", + response_valid=False, + error_kind="unsupported", + error_detail="scheme not probed", + ) + session.add(result) + await session.commit() + return result + + success = outcome.response_valid and outcome.status == "up" + if success: + tracker.consecutive_failures = 0 + tracker.current_status = "up" + if outcome.tracker_interval_seconds: + tracker.announced_interval_seconds = outcome.tracker_interval_seconds + else: + tracker.consecutive_failures += 1 + tracker.current_status = ( + outcome.status if outcome.status in {"down", "degraded"} else "down" + ) + + tracker.last_latency_ms = outcome.latency_ms + tracker.last_checked_at = now + tracker.next_check_at = compute_next_check( + settings=settings, + announced_interval=tracker.announced_interval_seconds, + consecutive_failures=tracker.consecutive_failures, + success=success, + ) + + result = ProbeResult( + tracker_id=tracker.id, + checked_at=now, + status=tracker.current_status, + latency_ms=outcome.latency_ms, + ip_address=outcome.ip_address, + ip_family=outcome.ip_family, + response_valid=outcome.response_valid, + tracker_interval_seconds=outcome.tracker_interval_seconds, + seeders=outcome.seeders, + leechers=outcome.leechers, + error_kind=outcome.error_kind, + error_detail=(outcome.error_detail or "")[:512] or None, + ) + session.add(result) + await session.commit() + return result + + +async def probe_due_batch( + session: AsyncSession, + settings: Settings, + *, + limit: int | None = None, +) -> int: + from app.db import get_session_factory + from app.db.queries import due_trackers + + limit = limit or settings.probe_concurrency + trackers = await due_trackers(session, limit=limit) + if not trackers: + return 0 + + ids = [t.id for t in trackers] + sem = asyncio.Semaphore(settings.probe_concurrency) + factory = get_session_factory() + + async with httpx.AsyncClient( + timeout=settings.probe_timeout_seconds, + headers={"User-Agent": settings.user_agent()}, + follow_redirects=False, + ) as client: + + async def _one(tracker_id: int) -> None: + async with sem, factory() as own_session: + tracker = await own_session.get(Tracker, tracker_id) + if tracker is None: + return + try: + await probe_tracker(own_session, tracker, settings, client) + except Exception: # noqa: BLE001 + logger.exception("probe failed for id=%s", tracker_id) + + await asyncio.gather(*[_one(i) for i in ids]) + return len(ids)
@@ -0,0 +1,156 @@
+"""Bencode decoder for tracker announce responses. + +Adapted from newTrackon (MIT). Rewritten for this architecture. +Peers are validated structurally and discarded by callers; this module never +persists peer addresses. +""" + +from __future__ import annotations + +from collections import OrderedDict +from socket import AF_INET, AF_INET6, inet_ntop +from struct import unpack_from +from typing import Any + +TOK_DICT = b"d" +TOK_LIST = b"l" +TOK_INT = b"i" +TOK_END = b"e" +TOK_STR_SEP = b":" + +BDecoded = OrderedDict[bytes, Any] | list[Any] | int | bytes | None + + +class BencodeError(ValueError): + pass + + +def bdecode(data: bytes) -> dict[str, Any]: + """Decode a bencoded tracker response into a string-keyed dict. + + Binary `peers` / `peers6` are converted to length-checked peer lists for + validation, then callers must discard addresses immediately. + """ + if not data: + raise BencodeError("empty bencode payload") + decoded = Decoder(data).decode() + if not isinstance(decoded, OrderedDict): + raise BencodeError("tracker response must be a dict") + response: dict[str, Any] = {} + for key, value in decoded.items(): + response[key.decode("utf-8", errors="replace")] = value + + if "peers" in response and isinstance(response["peers"], bytes): + response["peers"] = decode_binary_peers_list(response["peers"], 0, AF_INET) + if "peers6" in response and isinstance(response["peers6"], bytes): + response["peers6"] = decode_binary_peers_list(response["peers6"], 0, AF_INET6) + + for key, value in list(response.items()): + if isinstance(value, bytes): + response[key] = value.decode("utf-8", errors="replace") + return response + + +def decode_binary_peers_list(buf: bytes, offset: int, ip_family: int) -> list[dict[str, Any]]: + """Validate compact peer binary layout; return ephemeral peer dicts.""" + peers: list[dict[str, Any]] = [] + peer_length = 6 if ip_family == AF_INET else 18 + view = memoryview(buf) + while offset < len(buf): + if len(buf) < offset + peer_length: + break + ip_bytes = bytes(view[offset : offset + peer_length - 2]) + try: + ip_str = inet_ntop(ip_family, ip_bytes) + except OSError as exc: + raise BencodeError("invalid compact peer address") from exc + offset += peer_length - 2 + port = unpack_from("!H", buf, offset)[0] + offset += 2 + peers.append({"ip": ip_str, "port": port}) + return peers + + +def validate_peers_then_discard(response: dict[str, Any]) -> bool: + """Return True if peers/peers6 field exists and is structurally valid, then clear it.""" + has_peers = False + for key in ("peers", "peers6"): + if key not in response: + continue + value = response[key] + if isinstance(value, list): + has_peers = True + elif isinstance(value, (bytes, str)): + # dictionary-model peers — accept presence without storing + has_peers = True + response[key] = None + return has_peers + + +class Decoder: + def __init__(self, data: bytes) -> None: + self.index = 0 + self.data = data + + def decode(self) -> BDecoded: + c = self.peek() + if c is None: + raise BencodeError("unexpected EOF") + if c == TOK_DICT: + self.read(1) + return self.decode_dict() + if c == TOK_LIST: + self.read(1) + return self.decode_list() + if c == TOK_INT: + self.read(1) + return self.decode_int() + if c in b"0123456789": + return self.decode_str() + raise BencodeError("invalid bencode token") + + def peek(self) -> bytes | None: + if self.index >= len(self.data): + return None + return self.data[self.index : self.index + 1] + + def read(self, length: int) -> bytes: + if self.index + length > len(self.data): + raise BencodeError("unexpected EOF while reading") + result = self.data[self.index : self.index + length] + self.index += length + return result + + def read_until(self, token: bytes) -> bytes: + loc = self.data.find(token, self.index) + if loc == -1: + raise BencodeError("token not found") + result = self.data[self.index : loc] + self.index = loc + 1 + return result + + def decode_dict(self) -> OrderedDict[bytes, BDecoded]: + result: OrderedDict[bytes, BDecoded] = OrderedDict() + while self.data[self.index : self.index + 1] != TOK_END: + key = self.decode() + if not isinstance(key, bytes): + raise BencodeError("dict key must be bytes") + result[key] = self.decode() + self.read(1) + return result + + def decode_list(self) -> list[BDecoded]: + result: list[BDecoded] = [] + while self.data[self.index : self.index + 1] != TOK_END: + result.append(self.decode()) + self.read(1) + return result + + def decode_int(self) -> int: + return int(self.read_until(TOK_END)) + + def decode_str(self) -> bytes: + length = int(self.read_until(TOK_STR_SEP)) + if length < 0: + raise BencodeError("negative string length") + return self.read(length)
@@ -0,0 +1,169 @@
+from __future__ import annotations + +import os +import time + +import httpx + +from app.probes.bencode import BencodeError, bdecode, validate_peers_then_discard +from app.probes.types import ProbeOutcome +from app.services.ssrf import assert_safe_destination, is_public_ip + + +async def probe_http( + url: str, + *, + client: httpx.AsyncClient, + timeout: float = 10.0, + max_bytes: int = 1_048_576, + peer_id_prefix: bytes = b"-RASTRO-", + info_hash: bytes | None = None, + resolved_ips: list[str] | None = None, +) -> ProbeOutcome: + info_hash = info_hash or os.urandom(20) + peer_id = (peer_id_prefix[:8] + os.urandom(12))[:20] + from urllib.parse import quote_from_bytes + + query = ( + f"info_hash={quote_from_bytes(info_hash)}" + f"&peer_id={quote_from_bytes(peer_id)}" + f"&port=0&uploaded=0&downloaded=0&left=0&compact=1&numwant=0&event=started" + ) + announce_url = f"{url}?{query}" if "?" not in url else f"{url}&{query}" + + t0 = time.perf_counter() + try: + if resolved_ips is not None: + host = httpx.URL(url).host or "" + assert_safe_destination(host, resolved_ips) + + async with client.stream( + "GET", + announce_url, + timeout=timeout, + follow_redirects=False, + ) as response: + # Re-validate resolved addresses before consuming the body (anti-rebinding). + if resolved_ips: + for ip in resolved_ips: + if not is_public_ip(ip): + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="blocked", + error_detail=f"blocked address: {ip}", + latency_ms=(time.perf_counter() - t0) * 1000, + ) + + chunks: list[bytes] = [] + total = 0 + async for chunk in response.aiter_bytes(): + total += len(chunk) + if total > max_bytes: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="too_large", + error_detail="response exceeded MAX_RESPONSE_BYTES", + latency_ms=(time.perf_counter() - t0) * 1000, + ) + chunks.append(chunk) + body = b"".join(chunks) + latency = (time.perf_counter() - t0) * 1000 + + if response.status_code != 200: + return ProbeOutcome( + status="down", + response_valid=False, + latency_ms=latency, + error_kind="http_status", + error_detail=f"HTTP {response.status_code}", + ) + if not body: + return ProbeOutcome( + status="down", + response_valid=False, + latency_ms=latency, + error_kind="empty", + error_detail="empty HTTP response", + ) + try: + decoded = bdecode(body) + except BencodeError as exc: + return ProbeOutcome( + status="down", + response_valid=False, + latency_ms=latency, + error_kind="malformed", + error_detail=str(exc)[:512], + ) + + if "failure reason" in decoded: + return ProbeOutcome( + status="degraded", + response_valid=False, + latency_ms=latency, + error_kind="tracker_error", + error_detail=str(decoded["failure reason"])[:512], + ) + + # Require peers field OR interval for a minimal healthy announce + has_peers = validate_peers_then_discard(decoded) + interval = decoded.get("interval") + if not has_peers and interval is None: + return ProbeOutcome( + status="degraded", + response_valid=False, + latency_ms=latency, + error_kind="incomplete", + error_detail="missing peers and interval", + ) + + seeders = decoded.get("complete") + leechers = decoded.get("incomplete") + return ProbeOutcome( + status="up", + response_valid=True, + latency_ms=latency, + ip_address=resolved_ips[0] if resolved_ips else None, + ip_family="4" + if resolved_ips and ":" not in resolved_ips[0] + else ("6" if resolved_ips else None), + tracker_interval_seconds=int(interval) if isinstance(interval, int) else None, + seeders=int(seeders) if isinstance(seeders, int) else None, + leechers=int(leechers) if isinstance(leechers, int) else None, + ) + except httpx.TimeoutException: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="timeout", + error_detail="HTTP timeout", + latency_ms=(time.perf_counter() - t0) * 1000, + ) + except httpx.ConnectError as exc: + detail = str(exc) + kind = "tls" if "SSL" in detail.upper() or "TLS" in detail.upper() else "connection" + return ProbeOutcome( + status="down", + response_valid=False, + error_kind=kind, + error_detail=detail[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + ) + except ValueError as exc: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="blocked", + error_detail=str(exc)[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + ) + except httpx.HTTPError as exc: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="http", + error_detail=str(exc)[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + )
@@ -0,0 +1,17 @@
+from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(slots=True) +class ProbeOutcome: + status: str # up | down | degraded + response_valid: bool + latency_ms: float | None = None + ip_address: str | None = None + ip_family: str | None = None + tracker_interval_seconds: int | None = None + seeders: int | None = None + leechers: int | None = None + error_kind: str | None = None + error_detail: str | None = None
@@ -0,0 +1,170 @@
+"""UDP BitTorrent tracker probe (BEP 15). + +Protocol packing inspired by newTrackon (MIT). +""" + +from __future__ import annotations + +import asyncio +import os +import random +import socket +import struct +import time + +from app.probes.types import ProbeOutcome +from app.services.ssrf import assert_connected_ip_safe, assert_safe_destination + + +def _transaction_id() -> int: + return random.randint(0, 0x7FFFFFFF) + + +def create_connect_request() -> tuple[bytes, int]: + connection_id = 0x41727101980 + action = 0 + tid = _transaction_id() + buf = struct.pack("!qii", connection_id, action, tid) + return buf, tid + + +def parse_connect_response(buf: bytes, sent_tid: int) -> int: + if len(buf) < 16: + raise RuntimeError(f"connect response too short: {len(buf)}") + action, tid = struct.unpack_from("!ii", buf, 0) + if tid != sent_tid: + raise RuntimeError("transaction id mismatch on connect") + if action == 3: + raise RuntimeError("tracker returned UDP error on connect") + if action != 0: + raise RuntimeError(f"unexpected connect action: {action}") + return struct.unpack_from("!q", buf, 8)[0] + + +def create_announce_request( + connection_id: int, + info_hash: bytes, + peer_id: bytes, + *, + num_want: int = 0, + port: int = 0, +) -> tuple[bytes, int]: + action = 1 + tid = _transaction_id() + buf = struct.pack("!qii", connection_id, action, tid) + buf += struct.pack("!20s20s", info_hash, peer_id) + buf += struct.pack("!qqq", 0, 0, 0) # downloaded, left, uploaded + buf += struct.pack("!i", 2) # event = started + buf += struct.pack("!i", 0) # IP + buf += struct.pack("!i", _transaction_id()) # key + buf += struct.pack("!i", num_want) + buf += struct.pack("!H", port) + return buf, tid + + +def parse_announce_response(buf: bytes, sent_tid: int) -> tuple[int, int, int]: + """Return (interval, leechers, seeders). Peer bytes are ignored/discarded.""" + if len(buf) < 20: + raise RuntimeError(f"announce response too short: {len(buf)}") + action, tid = struct.unpack_from("!ii", buf, 0) + if tid != sent_tid: + raise RuntimeError("transaction id mismatch on announce") + if action == 3: + raise RuntimeError("tracker returned UDP error on announce") + if action != 1: + raise RuntimeError(f"unexpected announce action: {action}") + interval, leechers, seeders = struct.unpack_from("!iii", buf, 8) + # Remaining bytes would be peers — intentionally not parsed into storage. + return interval, leechers, seeders + + +async def probe_udp( + host: str, + port: int, + *, + timeout: float = 10.0, + info_hash: bytes | None = None, + peer_id: bytes | None = None, +) -> ProbeOutcome: + info_hash = info_hash or os.urandom(20) + peer_id = peer_id or (b"-RASTRO-" + os.urandom(12)) + if len(peer_id) != 20: + peer_id = (peer_id + os.urandom(20))[:20] + + loop = asyncio.get_running_loop() + t0 = time.perf_counter() + try: + infos = await loop.getaddrinfo(host, port, type=socket.SOCK_DGRAM) + except OSError as exc: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="dns", + error_detail=str(exc)[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + ) + + ips = [str(item[4][0]) for item in infos] + try: + assert_safe_destination(host, ips) + except ValueError as exc: + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="blocked", + error_detail=str(exc)[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + ) + + last_error = "udp announce failed" + for af, socktype, proto, _, sockaddr in infos: + ip = str(sockaddr[0]) + try: + assert_connected_ip_safe(ip) + except ValueError as exc: + last_error = str(exc) + continue + + sock = socket.socket(af, socktype, proto) + sock.setblocking(False) + try: + await asyncio.wait_for(loop.sock_connect(sock, sockaddr), timeout=timeout) + connect_req, connect_tid = create_connect_request() + await asyncio.wait_for(loop.sock_sendall(sock, connect_req), timeout=timeout) + connect_buf = await asyncio.wait_for(loop.sock_recv(sock, 2048), timeout=timeout) + connection_id = parse_connect_response(connect_buf, connect_tid) + + announce_req, announce_tid = create_announce_request( + connection_id, info_hash, peer_id, num_want=0 + ) + await asyncio.wait_for(loop.sock_sendall(sock, announce_req), timeout=timeout) + announce_buf = await asyncio.wait_for(loop.sock_recv(sock, 2048), timeout=timeout) + interval, leechers, seeders = parse_announce_response(announce_buf, announce_tid) + latency = (time.perf_counter() - t0) * 1000 + family = "4" if af == socket.AF_INET else "6" + return ProbeOutcome( + status="up", + response_valid=True, + latency_ms=latency, + ip_address=ip, + ip_family=family, + tracker_interval_seconds=interval, + seeders=seeders, + leechers=leechers, + ) + except TimeoutError: + last_error = "udp timeout" + except OSError as exc: + last_error = f"udp error: {exc}" + except RuntimeError as exc: + last_error = str(exc) + finally: + sock.close() + + return ProbeOutcome( + status="down", + response_valid=False, + error_kind="udp", + error_detail=last_error[:512], + latency_ms=(time.perf_counter() - t0) * 1000, + )
@@ -0,0 +1,100 @@
+from __future__ import annotations + +import asyncio +import logging + +from apscheduler.schedulers.asyncio import AsyncIOScheduler + +from app.collectors import import_all_sources +from app.config import Settings, get_settings +from app.db import get_session_factory +from app.db.queries import prune_probe_results +from app.probes import probe_due_batch + +logger = logging.getLogger(__name__) + +_scheduler: AsyncIOScheduler | None = None +_locks = { + "import": asyncio.Lock(), + "probe": asyncio.Lock(), + "prune": asyncio.Lock(), +} + + +async def job_import_sources() -> None: + if _locks["import"].locked(): + logger.warning("import job already running; skipping") + return + async with _locks["import"]: + settings = get_settings() + factory = get_session_factory() + async with factory() as session: + await import_all_sources(session, settings) + + +async def job_probe_due() -> None: + if _locks["probe"].locked(): + logger.warning("probe job already running; skipping") + return + async with _locks["probe"]: + settings = get_settings() + factory = get_session_factory() + async with factory() as session: + count = await probe_due_batch(session, settings) + logger.info("probed %s trackers", count) + + +async def job_prune() -> None: + if _locks["prune"].locked(): + return + async with _locks["prune"]: + settings = get_settings() + factory = get_session_factory() + async with factory() as session: + removed = await prune_probe_results(session, settings.raw_result_retention_days) + logger.info("pruned %s probe results", removed) + + +def start_scheduler(settings: Settings | None = None) -> AsyncIOScheduler: + global _scheduler + settings = settings or get_settings() + if _scheduler is not None: + return _scheduler + scheduler = AsyncIOScheduler(timezone="UTC") + scheduler.add_job( + job_import_sources, + "cron", + day_of_week=settings.import_day_of_week, + hour=settings.import_hour_utc, + minute=0, + id="import_sources", + max_instances=1, + coalesce=True, + ) + scheduler.add_job( + job_probe_due, + "interval", + seconds=60, + id="probe_due", + max_instances=1, + coalesce=True, + ) + scheduler.add_job( + job_prune, + "interval", + hours=24, + id="prune_results", + max_instances=1, + coalesce=True, + ) + scheduler.start() + _scheduler = scheduler + logger.info("scheduler started (single process / single worker required)") + return scheduler + + +def stop_scheduler() -> None: + global _scheduler + if _scheduler is not None: + _scheduler.shutdown(wait=False) + _scheduler = None
@@ -0,0 +1,123 @@
+from __future__ import annotations + +import hashlib +import logging +import socket +from dataclasses import dataclass, field + +import dns.asyncresolver + +from app.services.ssrf import is_public_ip + +logger = logging.getLogger(__name__) + +MAX_CNAME_DEPTH = 10 + + +@dataclass +class DnsSnapshot: + cname_chain: list[str] = field(default_factory=list) + terminal_cname: str | None = None + ipv4: list[str] = field(default_factory=list) + ipv6: list[str] = field(default_factory=list) + cycle_detected: bool = False + fingerprint: str | None = None + + +async def resolve_cname_chain(hostname: str, *, max_depth: int = MAX_CNAME_DEPTH) -> DnsSnapshot: + snapshot = DnsSnapshot() + resolver = dns.asyncresolver.Resolver() + current = hostname.rstrip(".").lower() + seen: set[str] = set() + + for _ in range(max_depth): + if current in seen: + snapshot.cycle_detected = True + break + seen.add(current) + try: + answer = await resolver.resolve(current, "CNAME") + except Exception: # noqa: BLE001 — end of CNAME chain + break + if not answer: + break + target = str(answer[0].target).rstrip(".").lower() + snapshot.cname_chain.append(target) + current = target + + if snapshot.cname_chain: + snapshot.terminal_cname = snapshot.cname_chain[-1] + + lookup_host = snapshot.terminal_cname or hostname.rstrip(".").lower() + snapshot.ipv4 = await _resolve_addresses(resolver, lookup_host, "A") + snapshot.ipv6 = await _resolve_addresses(resolver, lookup_host, "AAAA") + snapshot.fingerprint = compute_fingerprint( + snapshot.terminal_cname, snapshot.ipv4 + snapshot.ipv6 + ) + return snapshot + + +async def _resolve_addresses( + resolver: dns.asyncresolver.Resolver, hostname: str, rdtype: str +) -> list[str]: + try: + answer = await resolver.resolve(hostname, rdtype) + except Exception: # noqa: BLE001 + return [] + out: list[str] = [] + for rdata in answer: + ip = rdata.to_text() + if is_public_ip(ip): + out.append(ip) + return sorted(set(out)) + + +def compute_fingerprint(terminal_cname: str | None, public_ips: list[str]) -> str | None: + if terminal_cname: + material = f"cname:{terminal_cname.lower()}" + elif public_ips: + material = "ips:" + ",".join(sorted(set(public_ips))) + else: + return None + return hashlib.sha256(material.encode("utf-8")).hexdigest()[:32] + + +def host_has_ipv6() -> bool: + """Best-effort detection of local IPv6 connectivity.""" + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + try: + sock.connect(("2001:4860:4860::8888", 53)) + return True + finally: + sock.close() + except OSError: + return False + + +def lookup_asn(ip: str, db_path: str) -> tuple[int | None, str | None, str | None]: + """Optional MaxMind ASN lookup. Returns (asn, network_name, country_code).""" + if not db_path: + return None, None, None + try: + import geoip2.database # type: ignore + except ImportError: + return None, None, None + try: + with geoip2.database.Reader(db_path) as reader: + # Prefer ASN db; country may be unavailable in ASN-only files + try: + asn_resp = reader.asn(ip) + asn = asn_resp.autonomous_system_number + org = asn_resp.autonomous_system_organization + except Exception: # noqa: BLE001 + asn, org = None, None + country = None + try: + country = reader.country(ip).country.iso_code # type: ignore[attr-defined] + except Exception: # noqa: BLE001 + country = None + return asn, org, country + except Exception: # noqa: BLE001 + logger.debug("GeoIP lookup unavailable for %s", ip, exc_info=True) + return None, None, None
@@ -0,0 +1,125 @@
+from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse, urlunparse + +SUPPORTED_SCHEMES = frozenset({"udp", "http", "https"}) +UNSUPPORTED_SCHEMES = frozenset({"ws", "wss", "i2p", "ygg"}) + + +class NormalizationError(ValueError): + """Raised when a URL cannot be normalized into a supported tracker endpoint.""" + + +@dataclass(frozen=True, slots=True) +class NormalizedTracker: + canonical_url: str + scheme: str + hostname: str + port: int + path: str + unsupported: bool = False + + +def _default_port(scheme: str) -> int: + if scheme == "https": + return 443 + if scheme == "http": + return 80 + return 80 # udp commonly requires explicit port; fallback unused when port missing + + +def _idna_hostname(hostname: str) -> str: + hostname = hostname.strip().rstrip(".").lower() + if not hostname: + raise NormalizationError("missing hostname") + try: + return hostname.encode("idna").decode("ascii") + except UnicodeError as exc: + raise NormalizationError("invalid IDN hostname") from exc + + +def normalize_tracker_url(raw: str) -> NormalizedTracker: + """Normalize a tracker URL into a canonical endpoint identity.""" + if raw is None: + raise NormalizationError("empty input") + text = raw.strip() + if not text or text.startswith("#"): + raise NormalizationError("empty input") + + # Reject credentials early even before parse edge-cases + if "@" in text.split("://", 1)[-1].split("/", 1)[0]: + raise NormalizationError("credentials not allowed") + + parsed = urlparse(text) + scheme = (parsed.scheme or "").lower() + if not scheme: + raise NormalizationError("missing scheme") + + if scheme in UNSUPPORTED_SCHEMES or scheme.endswith(".i2p") or "ygg" in scheme: + host = _idna_hostname(parsed.hostname or "unsupported") + try: + port = parsed.port or 0 + except ValueError as exc: + raise NormalizationError("invalid port") from exc + return NormalizedTracker( + canonical_url=text, + scheme=scheme, + hostname=host, + port=port, + path=parsed.path or "/", + unsupported=True, + ) + + if scheme not in SUPPORTED_SCHEMES: + raise NormalizationError(f"unsupported scheme: {scheme}") + + if parsed.username or parsed.password: + raise NormalizationError("credentials not allowed") + + if not parsed.hostname: + raise NormalizationError("missing hostname") + + hostname = _idna_hostname(parsed.hostname) + + try: + port = parsed.port + except ValueError as exc: + raise NormalizationError("invalid port") from exc + if port is None: + # Prefer explicit ports when provided by source; otherwise defaults. + port = _default_port(scheme) + if not (1 <= port <= 65535): + raise NormalizationError("invalid port") + + path = parsed.path or "/announce" + if not path.startswith("/"): + path = "/" + path + # Drop fragment; query is uncommon for announce base URLs — strip for identity. + # Preserve path as given by source (including /announce variants). + + netloc = f"{hostname}:{port}" + canonical = urlunparse((scheme, netloc, path, "", "", "")) + return NormalizedTracker( + canonical_url=canonical, + scheme=scheme, + hostname=hostname, + port=port, + path=path, + unsupported=False, + ) + + +def parse_tracker_list_text(body: str) -> list[str]: + """Split a blank-line-tolerant tracker list into raw URL strings (deduped).""" + seen: set[str] = set() + out: list[str] = [] + for line in body.splitlines(): + item = line.strip() + if not item or item.startswith("#"): + continue + if item in seen: + continue + seen.add(item) + out.append(item) + return out
@@ -0,0 +1,151 @@
+from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from statistics import median + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ProbeResult, Tracker + + +@dataclass(slots=True) +class TrackerMetrics: + uptime_24h: float | None + uptime_7d: float | None + uptime_30d: float | None + latency_median_7d: float | None + latency_p95_7d: float | None + valid_rate_7d: float | None + measurement_count: int + measurement_count_7d: int + tracking_days: float + score: int | None + provisional: bool + + +def _percentile(sorted_values: list[float], p: float) -> float | None: + if not sorted_values: + return None + if len(sorted_values) == 1: + return sorted_values[0] + k = (len(sorted_values) - 1) * p + f = int(k) + c = min(f + 1, len(sorted_values) - 1) + if f == c: + return sorted_values[f] + return sorted_values[f] + (sorted_values[c] - sorted_values[f]) * (k - f) + + +def uptime_ratio(results: Sequence[ProbeResult]) -> float | None: + if not results: + return None + ups = sum(1 for r in results if r.status == "up" and r.response_valid) + return ups / len(results) + + +def latency_score(median_ms: float | None) -> int: + if median_ms is None: + return 0 + if median_ms <= 150: + return 20 + if median_ms <= 300: + return 16 + if median_ms <= 600: + return 10 + if median_ms <= 1000: + return 5 + return 0 + + +def confidence_score(count: int) -> int: + if count < 3: + return 0 + if count < 10: + return 1 + if count < 20: + return 3 + return 5 + + +def compute_score( + *, + uptime_7d: float | None, + latency_median_7d: float | None, + valid_rate_7d: float | None, + measurement_count: int, +) -> int | None: + if measurement_count < 3: + return None + avail = int(round((uptime_7d or 0.0) * 60)) + lat = latency_score(latency_median_7d) + valid = int(round((valid_rate_7d or 0.0) * 15)) + conf = confidence_score(measurement_count) + return max(0, min(100, avail + lat + valid + conf)) + + +def metrics_from_results( + tracker: Tracker, results: Sequence[ProbeResult], *, now: datetime | None = None +) -> TrackerMetrics: + now = now or datetime.now(UTC) + + # Normalize naive datetimes from SQLite + def _aware(dt: datetime) -> datetime: + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + + first_seen = _aware(tracker.first_seen_at) + cut_24h = now - timedelta(hours=24) + cut_7d = now - timedelta(days=7) + cut_30d = now - timedelta(days=30) + + aware_results = [] + for r in results: + # mutate view via checked_at comparison with aware times + checked = _aware(r.checked_at) + aware_results.append((checked, r)) + + r24 = [r for checked, r in aware_results if checked >= cut_24h] + r7 = [r for checked, r in aware_results if checked >= cut_7d] + r30 = [r for checked, r in aware_results if checked >= cut_30d] + + latencies = sorted(r.latency_ms for r in r7 if r.latency_ms is not None and r.response_valid) + valid_rate = None + if r7: + valid_rate = sum(1 for r in r7 if r.response_valid) / len(r7) + + count = len(results) + provisional = count < 10 + score = compute_score( + uptime_7d=uptime_ratio(r7), + latency_median_7d=float(median(latencies)) if latencies else None, + valid_rate_7d=valid_rate, + measurement_count=count, + ) + + tracking_days = (now - first_seen).total_seconds() / 86400.0 + return TrackerMetrics( + uptime_24h=uptime_ratio(r24), + uptime_7d=uptime_ratio(r7), + uptime_30d=uptime_ratio(r30), + latency_median_7d=float(median(latencies)) if latencies else None, + latency_p95_7d=_percentile(list(latencies), 0.95), + valid_rate_7d=valid_rate, + measurement_count=count, + measurement_count_7d=len(r7), + tracking_days=tracking_days, + score=score, + provisional=provisional, + ) + + +async def load_metrics(session: AsyncSession, tracker: Tracker) -> TrackerMetrics: + result = await session.execute( + select(ProbeResult) + .where(ProbeResult.tracker_id == tracker.id) + .order_by(ProbeResult.checked_at.desc()) + .limit(5000) + ) + rows = list(result.scalars().all()) + return metrics_from_results(tracker, rows)
@@ -0,0 +1,137 @@
+from __future__ import annotations + +from dataclasses import dataclass + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.db.models import ProbeResult, Tracker +from app.services.scoring import TrackerMetrics, load_metrics + + +@dataclass +class SmartListFilters: + protocol: str | None = None # udp|http|https + min_uptime: float = 0.95 + max_latency_ms: float | None = None + ip_version: str = "any" # any|ipv4|ipv6 + limit: int = 20 + min_age_days: float = 0.0 + diversity: bool = True + require_valid: bool = True + min_measurements: int = 10 + + +@dataclass +class SmartListEntry: + tracker: Tracker + metrics: TrackerMetrics + + +async def build_smart_list( + session: AsyncSession, filters: SmartListFilters | None = None +) -> list[SmartListEntry]: + filters = filters or SmartListFilters() + limit = max(1, min(50, filters.limit)) + + stmt = ( + select(Tracker).where(Tracker.current_status == "up").options(selectinload(Tracker.sources)) + ) + if filters.protocol: + stmt = stmt.where(Tracker.scheme == filters.protocol.lower()) + + result = await session.execute(stmt) + trackers = list(result.scalars().unique().all()) + + scored: list[SmartListEntry] = [] + for tracker in trackers: + metrics = await load_metrics(session, tracker) + if metrics.measurement_count < filters.min_measurements: + continue + if filters.require_valid and (metrics.valid_rate_7d or 0) <= 0: + # require at least some valid responses; last probe should be valid + last = await session.execute( + select(ProbeResult) + .where(ProbeResult.tracker_id == tracker.id) + .order_by(ProbeResult.checked_at.desc()) + .limit(1) + ) + last_row = last.scalar_one_or_none() + if last_row is None or not last_row.response_valid: + continue + if metrics.uptime_7d is None or metrics.uptime_7d < filters.min_uptime: + continue + if filters.max_latency_ms is not None and ( + metrics.latency_median_7d is None or metrics.latency_median_7d > filters.max_latency_ms + ): + continue + if metrics.tracking_days < filters.min_age_days: + continue + if filters.ip_version == "ipv4" and not tracker.supports_ipv4: + continue + if filters.ip_version == "ipv6" and tracker.supports_ipv6 != "true": + continue + scored.append(SmartListEntry(tracker=tracker, metrics=metrics)) + + scored.sort( + key=lambda e: ( + -(e.metrics.score or 0), + e.metrics.latency_median_7d if e.metrics.latency_median_7d is not None else 1e9, + ) + ) + + if not filters.diversity: + return scored[:limit] + + selected: list[SmartListEntry] = [] + seen_fps: set[str] = set() + asn_counts: dict[int, int] = {} + for entry in scored: + fp = entry.tracker.infrastructure_fingerprint + if fp and fp in seen_fps: + continue + asn = entry.tracker.asn + if asn is not None and asn_counts.get(asn, 0) >= 2: + continue + selected.append(entry) + if fp: + seen_fps.add(fp) + if asn is not None: + asn_counts[asn] = asn_counts.get(asn, 0) + 1 + if len(selected) >= limit: + break + return selected + + +def format_smart_list_txt(entries: list[SmartListEntry]) -> str: + if not entries: + return "" + return "\n\n".join(e.tracker.canonical_url for e in entries) + "\n" + + +def format_smart_list_json(entries: list[SmartListEntry]) -> list[dict]: + out = [] + for e in entries: + out.append( + { + "id": e.tracker.id, + "url": e.tracker.canonical_url, + "scheme": e.tracker.scheme, + "score": e.metrics.score, + "uptime_7d": e.metrics.uptime_7d, + "latency_median_7d": e.metrics.latency_median_7d, + "infrastructure_fingerprint": e.tracker.infrastructure_fingerprint, + "asn": e.tracker.asn, + } + ) + return out + + +async def build_live_list(session: AsyncSession) -> list[str]: + result = await session.execute( + select(Tracker) + .where(Tracker.current_status == "up") + .order_by(Tracker.last_latency_ms.asc().nullslast()) + ) + return [t.canonical_url for t in result.scalars().all()]
@@ -0,0 +1,78 @@
+from __future__ import annotations + +import ipaddress +from ipaddress import IPv4Address, IPv6Address + +# AWS/GCP/Azure-style metadata endpoints commonly abused for SSRF +METADATA_HOSTNAMES = frozenset( + { + "metadata.google.internal", + "metadata.goog", + "kubernetes.default", + "kubernetes.default.svc", + } +) + + +def is_blocked_hostname(hostname: str) -> bool: + host = hostname.strip(".").lower() + if host in METADATA_HOSTNAMES: + return True + if host.endswith(".local") or host.endswith(".localhost"): + return True + return host in {"localhost", "metadata"} + + +def is_public_ip(ip_str: str) -> bool: + """Return True if the IP is globally routable / safe to connect to.""" + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + return False + return not _is_blocked_ip(ip) + + +def _is_blocked_ip(ip: IPv4Address | IPv6Address) -> bool: + if ip.is_loopback: + return True + if ip.is_private: + return True + if ip.is_link_local: + return True + if ip.is_multicast: + return True + if ip.is_reserved: + return True + if ip.is_unspecified: + return True + # CGNAT / documentation / benchmarking often flagged via is_private or reserved; + # also block classic metadata IPv4 explicitly. + if isinstance(ip, IPv4Address): + if ip == IPv4Address("169.254.169.254"): + return True + if ip in ipaddress.ip_network("0.0.0.0/8"): + return True + if isinstance(ip, IPv6Address): + # Unique local addresses + if ip.ipv4_mapped is not None: + return _is_blocked_ip(ip.ipv4_mapped) + if (int(ip) >> 118) == 0b1111110: # fc00::/7 + return True + return False + + +def assert_safe_destination(hostname: str, resolved_ips: list[str]) -> None: + """Raise ValueError if hostname or any resolved IP is unsafe.""" + if is_blocked_hostname(hostname): + raise ValueError(f"blocked hostname: {hostname}") + if not resolved_ips: + raise ValueError("no resolved addresses") + for ip in resolved_ips: + if not is_public_ip(ip): + raise ValueError(f"blocked address: {ip}") + + +def assert_connected_ip_safe(ip_str: str) -> None: + """Re-check the address actually used for the connection (anti-rebinding).""" + if not is_public_ip(ip_str): + raise ValueError(f"blocked connected address: {ip_str}")
@@ -0,0 +1,346 @@
+/* Modernist-inspired theme (local recreation). No CDN. */ + +:root { + --bg-outer-top: #6c7989; + --bg-outer-bottom: #434b55; + --text: #555555; + --text-strong: #222222; + --muted: #61778b; + --link: #006699; + --header-top: #ddfbfc; + --header-bottom: #c6eafa; + --header-border: #b2d2e1; + --btn-top: #77b9fb; + --btn-bottom: #3782cd; + --btn-border: #3a7cbe; + --btn-shine: #8bbef3; + --section-top: #fafafa; + --section-bottom: #dedede; + --wrapper-bg: #dedede; + --up: #2a7a2a; + --down: #a33; + --degraded: #a67c00; + --unknown: #666; + --max-width: 1220px; +} + +* { box-sizing: border-box; } + +html { + background: var(--bg-outer-top); + background: linear-gradient(var(--bg-outer-top), var(--bg-outer-bottom)) fixed; + min-height: 100%; +} + +body { + margin: 0; + padding: 50px 0; + font: 14px/1.5 Lato, "Helvetica Neue", Helvetica, Arial, sans-serif; + color: var(--text); + font-weight: 300; + min-height: calc(100% - 100px); +} + +a { + color: var(--link); + text-decoration: none; +} +a:hover { text-decoration: underline; } +a:focus-visible, +button:focus-visible, +input:focus-visible, +select:focus-visible { + outline: 3px solid #ffbf47; + outline-offset: 2px; +} + +.brand { color: var(--link); } +.brand:hover { text-decoration: none; } + +.wrapper { + width: min(100% - 24px, var(--max-width)); + margin: 0 auto; + background: var(--wrapper-bg); + border-radius: 8px; + box-shadow: rgba(0, 0, 0, 0.2) 0 0 0 1px, rgba(0, 0, 0, 0.45) 0 3px 10px; +} + +header { + border-radius: 8px 8px 0 0; + background: linear-gradient(var(--header-top), var(--header-bottom)); + position: relative; + padding: 15px 20px; + border-bottom: 1px solid var(--header-border); + min-height: 78px; +} + +header h1 { + margin: 0; + padding: 0; + font-size: 24px; + line-height: 1.2; + color: var(--link); + text-shadow: rgba(255, 255, 255, 0.9) 0 1px 0; +} + +header > p { + margin: 0; + color: var(--muted); + max-width: 42ch; + font-size: 13px; +} + +.lang-switch { + margin: 6px 0 0; + font-size: 12px; + color: var(--muted); +} +.lang-switch a { color: var(--muted); } +.lang-switch strong { color: var(--text-strong); } + +.nav-seg { + margin: 0; + padding: 1px 0; + list-style: none; + position: absolute; + z-index: 1; + right: 20px; + top: 20px; + height: 38px; + background: linear-gradient(var(--btn-top), var(--btn-bottom)); + border-radius: 5px; + box-shadow: inset rgba(255, 255, 255, 0.45) 0 1px 0, inset rgba(0, 0, 0, 0.2) 0 -1px 0; + display: flex; +} + +.nav-seg li { + border-right: 1px solid var(--btn-border); +} +.nav-seg li:last-child { border-right: none; } +.nav-seg li + li { border-left: 1px solid var(--btn-shine); } + +.nav-seg a { + line-height: 1; + font-size: 11px; + color: rgba(255, 255, 255, 0.85); + display: block; + text-align: center; + font-weight: 400; + padding: 6px 12px 0; + height: 38px; + text-shadow: rgba(0, 0, 0, 0.4) 0 -1px 0; + text-decoration: none; + min-width: 72px; +} +.nav-seg a strong { + font-size: 14px; + display: block; + color: #fff; + font-weight: 700; +} +.nav-seg li.active a, +.nav-seg a:hover { + background: rgba(0, 0, 0, 0.08); +} + +section { + padding: 15px 20px 24px; + font-size: 15px; + border-top: 1px solid #fff; + background: linear-gradient(var(--section-top), var(--section-bottom) 700px); + border-radius: 0 0 8px 8px; +} + +h2, h3 { color: #393939; } +h2 { margin: 0 0 12px; } +h3 { margin: 20px 0 8px; } + +.mono, code, pre, .list-out { + font-family: Monaco, "Bitstream Vera Sans Mono", "Lucida Console", Terminal, monospace; +} + +.muted { color: var(--muted); font-size: 0.9em; } + +.stats-strip { + display: flex; + flex-wrap: wrap; + gap: 8px 16px; + margin: 0 0 16px; + padding: 8px 0; + border-bottom: 1px solid #aaa; + font-size: 13px; +} +.stats-strip strong { color: var(--text-strong); } + +.filters { + display: flex; + flex-wrap: wrap; + gap: 10px 14px; + align-items: end; + margin: 0 0 16px; +} +.filters label { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 12px; + color: var(--muted); +} +.filters input, +.filters select { + font: inherit; + padding: 4px 6px; + border: 1px solid #aaa; + border-radius: 3px; + background: #fff; + color: var(--text-strong); + min-width: 0; +} +.filters button, +button.copy-btn { + font: inherit; + padding: 6px 12px; + border: 1px solid var(--btn-border); + border-radius: 4px; + color: #fff; + background: linear-gradient(var(--btn-top), var(--btn-bottom)); + cursor: pointer; + text-shadow: rgba(0, 0, 0, 0.35) 0 -1px 0; +} +button.linkish { + background: none; + border: none; + color: var(--link); + cursor: pointer; + padding: 0 0 0 6px; + font: inherit; + font-size: 12px; + text-shadow: none; +} + +.table-wrap { + width: 100%; + overflow-x: auto; + -webkit-overflow-scrolling: touch; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} +th, td { + text-align: left; + padding: 6px 8px; + border-bottom: 1px solid #aaa; + vertical-align: top; +} +th { + color: var(--text-strong); + font-weight: 700; + background: rgba(255, 255, 255, 0.45); +} + +.status { white-space: nowrap; } +.status .dot { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 50%; + margin-right: 6px; + background: var(--unknown); + vertical-align: middle; +} +.status-up .dot { background: var(--up); } +.status-down .dot { background: var(--down); } +.status-degraded .dot { background: var(--degraded); } +.status-unsupported .dot { background: #888; } + +.badge { + display: inline-block; + margin-left: 6px; + padding: 1px 6px; + border: 1px solid #aaa; + border-radius: 3px; + font-size: 11px; + color: var(--muted); +} + +.notice { + background: rgba(0, 0, 0, 0.06); + padding: 10px 12px; + margin: 0 0 14px; +} + +.metrics-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: 8px 12px; + margin: 0 0 16px; +} + +.sparkline { + width: 100%; + max-width: 480px; + height: 48px; + margin: 0 0 16px; +} +.sparkline circle { fill: var(--link); } +.sparkline line { stroke: var(--link); stroke-width: 1.5; } + +.list-out, pre { + background: #3a3c42; + color: #f8f8f2; + padding: 16px; + overflow-x: auto; + white-space: pre-wrap; + word-break: break-all; + font-size: 12px; +} + +.pagination { + display: flex; + gap: 16px; + align-items: center; + margin-top: 12px; +} + +footer { + width: min(100% - 24px, var(--max-width)); + margin: 0 auto; + padding: 20px 0 0; + color: #ccc; + overflow: hidden; + display: flex; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + font-size: 13px; +} +footer a { color: #fff; font-weight: 700; } + +@media (max-width: 900px) { + body { padding: 0; } + .wrapper { + width: 100%; + border-radius: 0; + box-shadow: none; + } + header { border-radius: 0; padding-bottom: 60px; } + .nav-seg { + position: static; + margin-top: 12px; + flex-wrap: wrap; + height: auto; + } + .nav-seg a { height: auto; padding: 8px 10px; } + section { border-radius: 0; } + footer { width: auto; padding: 16px; } +} + +@media (prefers-reduced-motion: reduce) { + * { + animation: none !important; + transition: none !important; + } +}
@@ -0,0 +1,39 @@
+(function () { + "use strict"; + + console.log("hey stranger, nothing here. kisses, pablo"); + + function copyText(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + return navigator.clipboard.writeText(text); + } + var ta = document.createElement("textarea"); + ta.value = text; + document.body.appendChild(ta); + ta.select(); + try { + document.execCommand("copy"); + } finally { + document.body.removeChild(ta); + } + return Promise.resolve(); + } + + document.addEventListener("click", function (ev) { + var btn = ev.target.closest("[data-copy], [data-copy-target]"); + if (!btn) return; + var text = btn.getAttribute("data-copy"); + if (!text) { + var sel = btn.getAttribute("data-copy-target"); + var el = sel ? document.querySelector(sel) : null; + text = el ? el.textContent : ""; + } + copyText(text || "").then(function () { + var prev = btn.textContent; + btn.textContent = document.documentElement.getAttribute("data-copied") || "copied"; + setTimeout(function () { + btn.textContent = prev; + }, 1200); + }); + }); +})();
@@ -0,0 +1,44 @@
+<!DOCTYPE html> +<html lang="{{ html_lang }}" data-copied="{{ t('copied') }}"> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>{% block title %}{{ app_name }}{% endblock %}</title> + <meta name="description" content="{{ app_description }}"> + <link rel="stylesheet" href="/static/css/modernist.css"> +</head> +<body> + <div class="wrapper"> + <header> + <h1><a href="/" class="brand">{{ app_name }}</a></h1> + <p>{{ app_description }}</p> + <p class="lang-switch"> + {% if lang == 'pt' %}<strong>pt</strong>{% else %}<a href="/lang/pt">pt</a>{% endif %} + · + {% if lang == 'en' %}<strong>en</strong>{% else %}<a href="/lang/en">en</a>{% endif %} + </p> + <ul class="nav-seg"> + <li class="{% if active_nav == 'trackers' %}active{% endif %}"> + <a href="/"><strong>{{ t('nav_trackers') }}</strong>{{ t('nav_trackers_sub') }}</a> + </li> + <li class="{% if active_nav == 'lists' %}active{% endif %}"> + <a href="/lists"><strong>{{ t('nav_lists') }}</strong>{{ t('nav_lists_sub') }}</a> + </li> + {% if repository_url %} + <li> + <a href="{{ repository_url }}" rel="noopener noreferrer"><strong>{{ t('nav_github') }}</strong>{{ t('nav_github_sub') }}</a> + </li> + {% endif %} + </ul> + </header> + <section> + {% block content %}{% endblock %} + </section> + </div> + <footer> + <p>{{ app_name }} {{ app_version }} · {{ t('footer_utc') }}</p> + <p>{{ t('footer_disclaimer') }}</p> + </footer> + <script src="/static/js/app.js" defer></script> +</body> +</html>
@@ -0,0 +1,123 @@
+{% extends "base.html" %} +{% block title %}{{ app_name }} — {{ t('nav_trackers') }}{% endblock %} +{% block content %} +{% if bootstrap_note %} +<p class="notice" role="status">{{ bootstrap_note }}</p> +{% endif %} + +<div class="stats-strip" aria-label="{{ t('stats_label') }}"> + <span><strong>{{ stats.total }}</strong> {{ t('stats_monitored') }}</span> + <span><strong>{{ stats.up }}</strong> {{ t('stats_up') }}</span> + <span><strong>{{ stats.degraded }}</strong> {{ t('stats_degraded') }}</span> + <span><strong>{{ stats.down }}</strong> {{ t('stats_down') }}</span> + <span><strong>{{ stats.unknown }}</strong> {{ t('stats_unknown') }}</span> + <span>{{ t('stats_last_import') }}: {{ stats.last_import_at|dt }}</span> + <span>{{ t('stats_last_probe') }}: {{ stats.last_probe_at|dt }}</span> +</div> + +<form class="filters" method="get" action="/"> + <label>{{ t('filter_search') }} <input type="search" name="q" value="{{ q }}" placeholder="{{ t('filter_search_ph') }}"></label> + <label>{{ t('filter_protocol') }} + <select name="scheme"> + <option value="">{{ t('filter_any') }}</option> + <option value="udp" {% if scheme=='udp' %}selected{% endif %}>UDP</option> + <option value="http" {% if scheme=='http' %}selected{% endif %}>HTTP</option> + <option value="https" {% if scheme=='https' %}selected{% endif %}>HTTPS</option> + </select> + </label> + <label>{{ t('filter_status') }} + <select name="status"> + <option value="">{{ t('filter_any') }}</option> + {% for s in ['up','down','degraded','unknown','unsupported'] %} + <option value="{{ s }}" {% if status==s %}selected{% endif %}>{{ s }}</option> + {% endfor %} + </select> + </label> + <label>{{ t('filter_source') }} + <select name="source"> + <option value="">{{ t('filter_any') }}</option> + <option value="trackerslist" {% if source=='trackerslist' %}selected{% endif %}>trackerslist</option> + <option value="newtrackon" {% if source=='newtrackon' %}selected{% endif %}>newTrackon</option> + <option value="jokepool710" {% if source=='jokepool710' %}selected{% endif %}>jokepool710</option> + <option value="xiu2" {% if source=='xiu2' %}selected{% endif %}>xiu2</option> + </select> + </label> + <label>{{ t('filter_sort') }} + <select name="sort"> + <option value="checked" {% if sort=='checked' %}selected{% endif %}>{{ t('sort_checked') }}</option> + <option value="latency" {% if sort=='latency' %}selected{% endif %}>{{ t('sort_latency') }}</option> + <option value="url" {% if sort=='url' %}selected{% endif %}>{{ t('sort_url') }}</option> + <option value="status" {% if sort=='status' %}selected{% endif %}>{{ t('sort_status') }}</option> + <option value="protocol" {% if sort=='protocol' %}selected{% endif %}>{{ t('sort_protocol') }}</option> + </select> + </label> + <label>{{ t('filter_order') }} + <select name="order"> + <option value="desc" {% if order=='desc' %}selected{% endif %}>{{ t('order_desc') }}</option> + <option value="asc" {% if order=='asc' %}selected{% endif %}>{{ t('order_asc') }}</option> + </select> + </label> + <button type="submit">{{ t('filter_submit') }}</button> +</form> + +<div class="table-wrap"> + <table> + <thead> + <tr> + <th>{{ t('col_tracker') }}</th> + <th>{{ t('col_protocol') }}</th> + <th>{{ t('col_status') }}</th> + <th>{{ t('col_uptime7') }}</th> + <th>{{ t('col_latency') }}</th> + <th>{{ t('col_ip') }}</th> + <th>{{ t('col_asn') }}</th> + <th>{{ t('col_sources') }}</th> + <th>{{ t('col_aliases') }}</th> + <th>{{ t('col_last_check') }}</th> + </tr> + </thead> + <tbody> + {% for row in rows %} + <tr> + <td> + <a href="/trackers/{{ row.tracker.id }}" class="mono">{{ row.tracker.canonical_url }}</a> + <button type="button" class="linkish copy-btn" data-copy="{{ row.tracker.canonical_url }}">{{ t('copy') }}</button> + {% if row.metrics.provisional %}<span class="badge">{{ t('provisional') }}</span>{% endif %} + </td> + <td>{{ row.tracker.scheme|upper }}</td> + <td> + <span class="status status-{{ row.tracker.current_status }}" title="{{ row.tracker.current_status }}"> + <span class="dot" aria-hidden="true"></span>{{ row.tracker.current_status }} + </span> + </td> + <td>{{ row.metrics.uptime_7d|pct }}</td> + <td>{{ row.metrics.latency_median_7d|ms }}</td> + <td> + {% if row.tracker.supports_ipv4 %}v4{% else %}—{% endif %} + / + {% if row.tracker.supports_ipv6 == 'true' %}v6{% elif row.tracker.supports_ipv6 == 'not_tested' %}v6?{% else %}—{% endif %} + </td> + <td>{% if row.tracker.asn %}AS{{ row.tracker.asn }}{% else %}—{% endif %}{% if row.tracker.network_name %} {{ row.tracker.network_name }}{% endif %}</td> + <td>{{ row.sources or '—' }}</td> + <td>{{ row.alias_count }}</td> + <td>{{ row.tracker.last_checked_at|dt }}</td> + </tr> + {% else %} + <tr><td colspan="10">{{ t('empty_inventory') }}</td></tr> + {% endfor %} + </tbody> + </table> +</div> + +{% if pages > 1 %} +<nav class="pagination" aria-label="{{ t('pagination') }}"> + {% if page > 1 %} + <a href="?q={{ q }}&scheme={{ scheme }}&status={{ status }}&source={{ source }}&sort={{ sort }}&order={{ order }}&page={{ page-1 }}">{{ t('prev') }}</a> + {% endif %} + <span>{{ t('page_of', page=page, pages=pages, total=total) }}</span> + {% if page < pages %} + <a href="?q={{ q }}&scheme={{ scheme }}&status={{ status }}&source={{ source }}&sort={{ sort }}&order={{ order }}&page={{ page+1 }}">{{ t('next') }}</a> + {% endif %} +</nav> +{% endif %} +{% endblock %}
@@ -0,0 +1,52 @@
+{% extends "base.html" %} +{% block title %}{{ t('lists_title') }} — {{ app_name }}{% endblock %} +{% block content %} +<h2>{{ t('lists_heading') }}</h2> +<p>{{ t('lists_blurb') }}</p> + +<form class="filters" method="get" action="/lists"> + <label>{{ t('filter_protocol') }} + <select name="protocol"> + <option value="">{{ t('filter_any') }}</option> + <option value="udp" {% if protocol=='udp' %}selected{% endif %}>UDP</option> + <option value="http" {% if protocol=='http' %}selected{% endif %}>HTTP</option> + <option value="https" {% if protocol=='https' %}selected{% endif %}>HTTPS</option> + </select> + </label> + <label>{{ t('min_uptime') }} + <input type="number" name="min_uptime" step="0.01" min="0" max="1" value="{{ min_uptime }}"> + </label> + <label>{{ t('max_latency') }} + <input type="number" name="max_latency" min="1" value="{{ max_latency }}"> + </label> + <label>{{ t('filter_ip') }} + <select name="ip"> + <option value="any" {% if ip=='any' %}selected{% endif %}>{{ t('filter_any') }}</option> + <option value="ipv4" {% if ip=='ipv4' %}selected{% endif %}>IPv4</option> + <option value="ipv6" {% if ip=='ipv6' %}selected{% endif %}>IPv6</option> + </select> + </label> + <label>{{ t('filter_limit') }} + <input type="number" name="limit" min="1" max="50" value="{{ limit }}"> + </label> + <label>{{ t('min_age') }} + <input type="number" name="min_age_days" min="0" step="0.1" value="{{ min_age_days }}"> + </label> + <label>{{ t('diversity') }} + <select name="diversity"> + <option value="true" {% if diversity %}selected{% endif %}>{{ t('diversity_on') }}</option> + <option value="false" {% if not diversity %}selected{% endif %}>{{ t('diversity_off') }}</option> + </select> + </label> + <button type="submit">{{ t('generate') }}</button> +</form> + +<p><strong>{{ entries|length }}</strong> {{ t('selected_count') }}</p> +<p> + <button type="button" class="copy-btn" data-copy-target="#list-out">{{ t('copy_btn') }}</button> + <a href="{{ api_txt }}">{{ t('download_txt') }}</a> · + <a href="{{ api_json }}">{{ t('open_json') }}</a> +</p> +<p class="muted mono">{{ t('permalink') }}: {{ api_txt }}</p> +<pre id="list-out" class="list-out">{{ txt }}</pre> +{% endblock %}
@@ -0,0 +1,101 @@
+{% extends "base.html" %} +{% block title %}{{ tracker.canonical_url }} — {{ app_name }}{% endblock %} +{% block content %} +<p><a href="/">← {{ t('back') }}</a></p> +<h2 class="mono">{{ tracker.canonical_url }}</h2> +<p> + <span class="status status-{{ tracker.current_status }}"> + <span class="dot" aria-hidden="true"></span>{{ tracker.current_status }} + </span> + · {{ t('score') }}: + {% if metrics.score is not none %}{{ metrics.score }}{% else %}{{ t('score_insufficient') }}{% endif %} + {% if metrics.provisional %}<span class="badge">{{ t('provisional') }}</span>{% endif %} +</p> + +<div class="metrics-grid"> + <div><strong>{{ t('uptime_24h') }}</strong> {{ metrics.uptime_24h|pct }}</div> + <div><strong>{{ t('uptime_7d') }}</strong> {{ metrics.uptime_7d|pct }}</div> + <div><strong>{{ t('uptime_30d') }}</strong> {{ metrics.uptime_30d|pct }}</div> + <div><strong>{{ t('latency_med_7d') }}</strong> {{ metrics.latency_median_7d|ms }}</div> + <div><strong>{{ t('latency_p95') }}</strong> {{ metrics.latency_p95_7d|ms }}</div> + <div><strong>{{ t('valid_rate') }}</strong> {{ metrics.valid_rate_7d|pct }}</div> + <div><strong>{{ t('measurements') }}</strong> {{ metrics.measurement_count }}</div> + <div><strong>{{ t('tracking') }}</strong> {{ '%.1f'|format(metrics.tracking_days) }} {{ t('days') }}</div> +</div> + +{% if spark %} +<svg class="sparkline" viewBox="0 0 240 40" role="img" aria-label="{{ t('spark_label') }}"> + {% set mx = spark|max if spark|max > 0 else 1 %} + {% for v in spark %} + {% set x = loop.index0 * (240 / ([spark|length - 1, 1]|max)) %} + {% set y = 38 - (v / mx * 34) %} + <circle cx="{{ '%.1f'|format(x) }}" cy="{{ '%.1f'|format(y) }}" r="2" /> + {% if not loop.first %} + {% set px = (loop.index0 - 1) * (240 / ([spark|length - 1, 1]|max)) %} + {% set py = 38 - (spark[loop.index0 - 1] / mx * 34) %} + <line x1="{{ '%.1f'|format(px) }}" y1="{{ '%.1f'|format(py) }}" x2="{{ '%.1f'|format(x) }}" y2="{{ '%.1f'|format(y) }}" /> + {% endif %} + {% endfor %} +</svg> +{% endif %} + +<h3>{{ t('sources') }}</h3> +<ul> + {% for s in tracker.sources %} + <li> + {{ s.source_name }} + {% if s.is_best_at_source %}· {{ t('best_at_source') }}{% endif %} + {% if s.is_stable_at_source %}· {{ t('stable_at_source') }}{% endif %} + {% if s.is_live_at_source %}· {{ t('live_at_source') }}{% endif %} + <span class="muted">({{ t('seen') }} {{ s.last_seen_at|dt }})</span> + </li> + {% else %} + <li>{{ t('no_source') }}</li> + {% endfor %} +</ul> + +<h3>{{ t('dns') }}</h3> +<p>{{ t('terminal_cname') }}: <code>{{ tracker.terminal_cname or '—' }}</code></p> +<p>{{ t('infra_fp') }}: <code>{{ tracker.infrastructure_fingerprint or '—' }}</code></p> +<p>{{ t('asn') }}: {% if tracker.asn %}AS{{ tracker.asn }} {{ tracker.network_name or '' }}{% else %}{{ t('unknown') }}{% endif %} + · {{ t('country') }}: {{ tracker.country_code or t('unknown') }}</p> +<ul> + {% for d in tracker.dns_records %} + <li><code>{{ d.record_type }}</code> {{ d.value }}</li> + {% else %} + <li>{{ t('no_dns') }}</li> + {% endfor %} +</ul> + +{% if aliases %} +<h3>{{ t('shared_infra') }}</h3> +<p>{{ t('shared_infra_blurb') }}</p> +<ul> + {% for a in aliases %} + <li><a class="mono" href="/trackers/{{ a.id }}">{{ a.canonical_url }}</a></li> + {% endfor %} +</ul> +{% endif %} + +<h3>{{ t('recent_history') }}</h3> +<div class="table-wrap"> + <table> + <thead> + <tr><th>{{ t('when') }}</th><th>{{ t('col_status') }}</th><th>{{ t('th_latency') }}</th><th>{{ t('valid') }}</th><th>{{ t('error') }}</th></tr> + </thead> + <tbody> + {% for p in probes %} + <tr> + <td>{{ p.checked_at|dt }}</td> + <td>{{ p.status }}</td> + <td>{{ p.latency_ms|ms }}</td> + <td>{{ t('yes') if p.response_valid else t('no') }}</td> + <td>{% if p.error_kind %}{{ p.error_kind }}{% if p.error_detail %}: {{ p.error_detail }}{% endif %}{% else %}—{% endif %}</td> + </tr> + {% else %} + <tr><td colspan="5">{{ t('no_probes') }}</td></tr> + {% endfor %} + </tbody> + </table> +</div> +{% endblock %}
@@ -0,0 +1,24 @@
+services: + rastro: + build: . + restart: unless-stopped + # All configuration lives in .env (single source of truth for prod). + env_file: .env + volumes: + - ./data:/app/data + ports: + # host 10359 (localhost only) -> container 8090; reverse proxy fronts TLS + - "127.0.0.1:10359:8090" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/healthz', timeout=3)", + ] + interval: 30s + timeout: 5s + retries: 3 + start_period: 25s + stop_grace_period: 20s
@@ -0,0 +1,60 @@
+# Arquitetura + +## Visão geral + +Aplicação self-hosted de processo único que: + +1. importa trackers públicos de fontes externas; +2. normaliza e protege destinos; +3. sonda HTTP(S) e UDP; +4. persiste histórico em SQLite; +5. expõe HTML, TXT e JSON. + +Não é um tracker BitTorrent, não indexa torrents e não aceita magnets. + +## Processo único e Uvicorn + +O site, a API e o APScheduler rodam no **mesmo processo**. O Uvicorn deve ser +iniciado com **um único worker**. Múltiplos workers duplicariam jobs de importação +e probe. Essa limitação é intencional na versão `0.1.0`. + +Locks em memória impedem sobreposição do mesmo job dentro do processo. + +## Stack + +- Python 3.13, FastAPI, Jinja2, SQLAlchemy 2 async, Alembic, aiosqlite +- SQLite com WAL +- httpx, dnspython, APScheduler, orjson, pydantic-settings +- Docker de um único serviço; reverse proxy TLS fica fora (Nginx/CloudPanel) + +## Persistência + +Identidade histórica do tracker é preservada mesmo quando a URL some de uma +origem. A limpeza automática remove apenas `probe_results` expirados +(`RAW_RESULT_RETENTION_DAYS`). + +Datas no banco: UTC. API: ISO 8601. UI: apresentação humana com fuso indicado. + +## Segurança de rede + +Endereços ingeridos são não confiáveis. Destinos loopback, privados, link-local, +multicast, reservados e metadata são rejeitados. IPs do DNS são revalidados antes +da conexão (mitigação de rebinding). Respostas têm limite de bytes e timeout. +Não há endpoint público para testar URL arbitrária. + +## Configuração + +Nome, descrição, versão, URL pública e repositório vêm de variáveis de ambiente +(`APP_NAME`, etc.). O nome do produto não deve ser espalhado como literal no +código de domínio. + +## Degradação segura + +- Sem IPv6 no host: `ENABLE_IPV6=auto` marca IPv6 como não testado +- Sem banco GeoIP: ASN/país ficam desconhecidos +- Sem rede na primeira subida: site sobe vazio e registra o motivo + +## Extensões futuras (não implementadas) + +Contas, admin, submit público, probes distribuídos, PostgreSQL, filas, mapa, +notificações, protocolos WS/I2P/Yggdrasil.
@@ -0,0 +1,68 @@
+# Fontes externas + +Este documento descreve as fontes consumidas em tempo de execução. Listas de +origem **nunca** são apresentadas como trabalho original deste projeto. + +## 1. trackerslist + +- Repositório: <https://github.com/ngosang/trackerslist> +- Licença: GPLv2 (apenas consumo de dados publicados; nenhum código GPLv2) +- Arquivos: + - Principal: `https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt` + - Marcador “best”: `https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_best.txt` +- Formato: uma URL por linha, frequentemente com linha em branco entre entradas +- Papel no sistema: seed de endpoints públicos; `trackers_best.txt` só marca + `is_best_at_source` + +## 2. newTrackon + +- Repositório: <https://github.com/CorralPeltzer/newTrackon> +- Licença: MIT +- API: + - Principal: `https://newtrackon.com/api/all` + - Enrichment: `/api/stable`, `/api/live` (e rotas auxiliares se necessário) +- Formato: texto plano, URLs separadas por linhas em branco +- Papel: seed e marcadores `is_stable_at_source` / `is_live_at_source` +- Classificações da origem **não substituem** medições próprias + +## 3. jokepool710/Torrent-_Trackers + +- Repositório: <https://github.com/jokepool710/Torrent-_Trackers> +- Licença: MIT (© J0KEP00L, 2025) +- Arquivo consumido: + - `https://raw.githubusercontent.com/jokepool710/Torrent-_Trackers/main/trackers_all.txt` +- Formato: uma URL por linha +- Papel: seed independente de endpoints públicos. Os arquivos por protocolo + (`trackers_udp`, `trackers_http_https`, `trackers_ws_wss`) são subconjuntos do + agregado e **não** são consumidos. + +## 4. XIU2/TrackersListCollection + +- Repositório: <https://github.com/XIU2/TrackersListCollection> +- Licença: GPL-3.0 (apenas consumo de dados publicados; nenhum código GPL-3.0) +- Arquivos: + - Principal: `https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt` + - Marcador “best”: `https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/best.txt` +- Formato: URLs separadas por linhas em branco +- Papel: seed agregado de vários upstreams; `best.txt` só marca + `is_best_at_source`. Por ser agregador, há sobreposição esperada com as demais + fontes — a deduplicação por URL canônica evita contagem dupla e a medição + permanece própria e independente. + +## 5. Fontes avaliadas e não adotadas + +- **weibone/trackers_list** (GPL-2.0): publica o mesmo conjunto de arquivos que a + ngosang e se comporta como espelho diário; não adiciona endpoints novos. + +## 6. Distinção obrigatória + +| Tipo de dado | Origem | Significado | +|---|---|---| +| Proveniência / best / stable / live | Fontes externas | Metadado de ingestão | +| Status, latência, uptime, score | Medições locais | Resultado próprio | + +## 7. Política de falha + +Se uma origem estiver indisponível ou retornar conteúdo inválido/vazio, os dados +anteriores são preservados. Uma resposta vazia **nunca** é interpretada como +ordem para apagar o inventário.
@@ -0,0 +1,62 @@
+[project] +name = "rastro" +version = "0.1.0" +description = "Observatorio publico de trackers BitTorrent" +readme = "README.md" +license = { text = "MIT" } +authors = [{ name = "Pablo Murad" }] +requires-python = ">=3.13" +dependencies = [ + "fastapi>=0.115.0", + "uvicorn[standard]>=0.32.0", + "jinja2>=3.1.4", + "sqlalchemy[asyncio]>=2.0.36", + "aiosqlite>=0.20.0", + "alembic>=1.14.0", + "httpx>=0.28.0", + "dnspython>=2.7.0", + "apscheduler>=3.10.4", + "orjson>=3.10.0", + "pydantic-settings>=2.6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.0", + "pytest-asyncio>=0.24.0", + "respx>=0.21.0", + "ruff>=0.8.0", + "geoip2>=4.8.0", +] + +[project.scripts] +rastro = "app.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["app"] + +[tool.ruff] +target-version = "py313" +line-length = 100 +src = ["app", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] +ignore = ["B008"] + +[tool.ruff.lint.per-file-ignores] +"alembic/*" = ["E501"] +"tests/*" = ["E501"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +testpaths = ["tests"] +filterwarnings = ["ignore::DeprecationWarning"] + +[tool.ruff.format] +quote-style = "double"
@@ -0,0 +1,70 @@
+"""Shared pytest fixtures.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncGenerator +from pathlib import Path + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +# Ensure test settings before app import +os.environ.setdefault("APP_NAME", "RastroTest") +os.environ.setdefault("RUN_SCHEDULER", "false") +os.environ.setdefault("ENABLE_IPV6", "false") + +from app.config import Settings, get_settings +from app.db import reset_engine +from app.db.models import Base + + +@pytest.fixture +def settings(tmp_path: Path) -> Settings: + get_settings.cache_clear() + reset_engine() + db = tmp_path / "test.db" + s = Settings( + APP_NAME="RastroTest", + APP_DESCRIPTION="test", + APP_VERSION="0.1.0", + PUBLIC_URL="http://test.local", + DATABASE_URL=f"sqlite+aiosqlite:///{db.as_posix()}", + RUN_SCHEDULER=False, + ENABLE_IPV6="false", + ) + return s + + +@pytest_asyncio.fixture +async def session(settings: Settings) -> AsyncGenerator[AsyncSession]: + engine = create_async_engine(settings.database_url, echo=False) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as sess: + yield sess + await engine.dispose() + + +@pytest_asyncio.fixture +async def client( + settings: Settings, monkeypatch: pytest.MonkeyPatch +) -> AsyncGenerator[AsyncClient]: + get_settings.cache_clear() + reset_engine() + monkeypatch.setenv("DATABASE_URL", settings.database_url) + monkeypatch.setenv("RUN_SCHEDULER", "false") + monkeypatch.setenv("APP_NAME", "RastroTest") + get_settings.cache_clear() + + from app.main import create_app + + application = create_app() + transport = ASGITransport(app=application) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + get_settings.cache_clear() + reset_engine()
@@ -0,0 +1,3 @@
+udp://fixture.example:80/announce + +http://fixture.example:80/announce
@@ -0,0 +1,131 @@
+"""API / HTML integration tests (offline).""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest +from app.config import get_settings +from app.db import reset_engine +from app.db.models import Base, Tracker +from app.main import create_app +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + +@pytest.fixture +async def app_client(tmp_path, monkeypatch): + db = tmp_path / "api.db" + url = f"sqlite+aiosqlite:///{db.as_posix()}" + monkeypatch.setenv("DATABASE_URL", url) + monkeypatch.setenv("RUN_SCHEDULER", "false") + monkeypatch.setenv("APP_NAME", "RastroTest") + monkeypatch.setenv("ENABLE_IPV6", "false") + get_settings.cache_clear() + reset_engine() + + engine = create_async_engine(url) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as session: + session.add( + Tracker( + canonical_url="udp://demo.example:6969/announce", + scheme="udp", + hostname="demo.example", + port=6969, + path="/announce", + current_status="up", + first_seen_at=datetime.now(UTC), + last_seen_at=datetime.now(UTC), + last_checked_at=datetime.now(UTC), + last_latency_ms=42, + consecutive_failures=0, + infrastructure_fingerprint="abc", + supports_ipv4=True, + supports_ipv6="not_tested", + ) + ) + await session.commit() + await engine.dispose() + + # Patch lifespan import to skip network bootstrap + application = create_app() + transport = ASGITransport(app=application) + async with AsyncClient(transport=transport, base_url="http://test") as client: + yield client + get_settings.cache_clear() + reset_engine() + + +@pytest.mark.asyncio +async def test_health_meta_and_pages(app_client: AsyncClient): + r = await app_client.get("/healthz") + assert r.status_code == 200 + assert r.text == "ok" + + meta = await app_client.get("/api/v1/meta") + assert meta.status_code == 200 + body = meta.json() + assert body["name"] == "RastroTest" + assert "version" in body + + home = await app_client.get("/") + assert home.status_code == 200 + assert "demo.example" in home.text + assert "RastroTest" in home.text + + trackers = await app_client.get("/api/v1/trackers") + assert trackers.status_code == 200 + assert trackers.json()["total"] >= 1 + + detail = await app_client.get("/api/v1/trackers/1") + assert detail.status_code == 200 + + lists = await app_client.get("/lists") + assert lists.status_code == 200 + + +@pytest.mark.asyncio +async def test_pages_follow_language(app_client: AsyncClient): + home_en = await app_client.get("/", headers={"Accept-Language": "en-US,en;q=0.9"}) + assert home_en.status_code == 200 + assert 'lang="en"' in home_en.text + assert "watched" in home_en.text + + home_pt = await app_client.get( + "/", + headers={"Accept-Language": "en"}, + cookies={"lang": "pt"}, + ) + assert 'lang="pt-BR"' in home_pt.text + assert "monitorados" in home_pt.text + + missing_en = await app_client.get("/trackers/99999", headers={"Accept-Language": "en"}) + assert missing_en.status_code == 404 + assert "not found" in missing_en.text.lower() + + missing_pt = await app_client.get("/trackers/99999", cookies={"lang": "pt"}) + assert "não encontrado" in missing_pt.text + + +@pytest.mark.asyncio +async def test_lang_switch_sets_cookie(app_client: AsyncClient): + r = await app_client.get("/lang/en", follow_redirects=False) + assert r.status_code == 303 + assert "lang=en" in r.headers.get("set-cookie", "") + + +@pytest.mark.asyncio +async def test_smart_list_etag(app_client: AsyncClient): + first = await app_client.get("/api/v1/lists/smart.txt") + assert first.status_code == 200 + etag = first.headers.get("etag") + assert etag + second = await app_client.get("/api/v1/lists/smart.txt", headers={"If-None-Match": etag}) + assert second.status_code == 304 + + live = await app_client.get("/api/v1/lists/live.txt") + assert live.status_code == 200 + assert "demo.example" in live.text
@@ -0,0 +1,64 @@
+"""Bencode and UDP packet unit tests.""" + +from __future__ import annotations + +import struct + +import pytest +from app.probes.bencode import BencodeError, bdecode, validate_peers_then_discard +from app.probes.udp import ( + create_announce_request, + create_connect_request, + parse_announce_response, + parse_connect_response, +) + + +def test_bencode_valid_and_invalid(): + # d8:intervali1800e5:peers6:\x01\x02\x03\x04\x00Pe + peers = b"\x01\x02\x03\x04\x00\x50" + payload = b"d8:intervali1800e5:peers6:" + peers + b"e" + decoded = bdecode(payload) + assert decoded["interval"] == 1800 + assert isinstance(decoded["peers"], list) + assert validate_peers_then_discard(decoded) is True + assert decoded["peers"] is None + + with pytest.raises(BencodeError): + bdecode(b"") + with pytest.raises(BencodeError): + bdecode(b"i123") # incomplete + with pytest.raises(BencodeError): + bdecode(b"4:spam") # not a dict root for tracker helper — actually bdecode requires dict + # 4:spam is a string root + with pytest.raises(BencodeError): + bdecode(b"l4:spame") + + +def test_udp_connect_and_announce_roundtrip_fields(): + req, tid = create_connect_request() + assert len(req) == 16 + # forge response + conn_id = 0x1122334455667788 + resp = struct.pack("!iiq", 0, tid, conn_id) + assert parse_connect_response(resp, tid) == conn_id + + info = b"a" * 20 + peer = b"b" * 20 + ann, atid = create_announce_request(conn_id, info, peer, num_want=0) + assert len(ann) == 98 + # announce response: action, tid, interval, leechers, seeders + fake peers + aresp = struct.pack("!iiiii", 1, atid, 1800, 2, 5) + b"\x00" * 6 + interval, leechers, seeders = parse_announce_response(aresp, atid) + assert interval == 1800 + assert leechers == 2 + assert seeders == 5 + + +def test_udp_transaction_mismatch(): + resp = struct.pack("!iiq", 0, 123, 99) + with pytest.raises(RuntimeError, match="transaction"): + parse_connect_response(resp, 456) + aresp = struct.pack("!iiiii", 1, 1, 10, 0, 0) + with pytest.raises(RuntimeError, match="transaction"): + parse_announce_response(aresp, 2)
@@ -0,0 +1,96 @@
+"""Collector parsing and failure isolation tests.""" + +from __future__ import annotations + +import httpx +import pytest +import respx +from app.collectors.base import parse_and_normalize +from app.collectors.trackerslist import TrackerslistCollector +from app.config import Settings +from app.db.models import Tracker, TrackerSource +from sqlalchemy import select + +FIXTURE_ALL = """ +udp://tracker.example:6969/announce + +http://tracker.example:80/announce + +udp://tracker.example:6969/announce + +ws://ignored.example/announce +""" + +FIXTURE_BEST = """ +udp://tracker.example:6969/announce +""" + + +def test_parse_and_normalize_counts(): + parsed = parse_and_normalize(FIXTURE_ALL) + assert parsed.received == 3 + assert len(parsed.valid) == 2 + assert len(parsed.unsupported) == 1 + + +@pytest.mark.asyncio +@respx.mock +async def test_trackerslist_import_and_empty_does_not_wipe(session, settings: Settings): + settings = Settings( + APP_NAME="RastroTest", + DATABASE_URL=settings.database_url, + TRACKERSLIST_ALL_URL="https://example.test/all.txt", + TRACKERSLIST_BEST_URL="https://example.test/best.txt", + RUN_SCHEDULER=False, + ) + respx.get("https://example.test/all.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_ALL) + ) + respx.get("https://example.test/best.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_BEST) + ) + collector = TrackerslistCollector() + async with httpx.AsyncClient() as client: + stats = await collector.import_once(session, client, settings) + assert stats.valid == 2 + assert stats.new == 2 + assert stats.error is None + + rows = (await session.execute(select(Tracker))).scalars().all() + assert len(rows) == 2 + sources = (await session.execute(select(TrackerSource))).scalars().all() + assert any(s.is_best_at_source for s in sources) + + # Failed empty import must preserve data + respx.get("https://example.test/all.txt").mock(return_value=httpx.Response(200, text="\n\n")) + async with httpx.AsyncClient() as client: + stats2 = await collector.import_once(session, client, settings) + assert stats2.error is not None + rows2 = (await session.execute(select(Tracker))).scalars().all() + assert len(rows2) == 2 + + +@pytest.mark.asyncio +@respx.mock +async def test_source_failure_preserves_previous(session, settings: Settings): + settings = Settings( + APP_NAME="RastroTest", + DATABASE_URL=settings.database_url, + TRACKERSLIST_ALL_URL="https://example.test/all.txt", + TRACKERSLIST_BEST_URL="https://example.test/best.txt", + RUN_SCHEDULER=False, + ) + respx.get("https://example.test/all.txt").mock( + return_value=httpx.Response(200, text="udp://ok.example:80/announce\n") + ) + respx.get("https://example.test/best.txt").mock(return_value=httpx.Response(200, text="")) + collector = TrackerslistCollector() + async with httpx.AsyncClient() as client: + await collector.import_once(session, client, settings) + + respx.get("https://example.test/all.txt").mock(return_value=httpx.Response(503)) + async with httpx.AsyncClient() as client: + stats = await collector.import_once(session, client, settings) + assert stats.error + count = len((await session.execute(select(Tracker))).scalars().all()) + assert count == 1
@@ -0,0 +1,79 @@
+"""DNS CNAME chain / cycle unit tests with mocked resolver.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from app.services.dns import compute_fingerprint, resolve_cname_chain + + +class _Rdata: + def __init__(self, target: str): + self.target = target + + def to_text(self) -> str: + return self.target + + +@pytest.mark.asyncio +async def test_cname_chain_and_cycle(): + calls = {"n": 0} + + async def fake_resolve(name, rdtype): + calls["n"] += 1 + host = str(name).rstrip(".").lower() if not isinstance(name, str) else name.lower() + # dnspython passes string hostname in our code + host = name if isinstance(name, str) else str(name) + host = host.rstrip(".").lower() + if rdtype == "CNAME": + mapping = { + "a.example": "b.example", + "b.example": "a.example", # cycle + } + if host in mapping: + return [_Rdata(mapping[host])] + raise Exception("no cname") + if rdtype == "A": + return [] + if rdtype == "AAAA": + return [] + if rdtype == "TXT": + raise Exception("no txt") + raise Exception("unexpected") + + with patch("app.services.dns.dns.asyncresolver.Resolver") as resolver_cls: + inst = resolver_cls.return_value + inst.resolve = AsyncMock(side_effect=fake_resolve) + snap = await resolve_cname_chain("a.example") + assert snap.cycle_detected is True + assert "b.example" in snap.cname_chain + + +@pytest.mark.asyncio +async def test_cname_terminal_fingerprint(): + async def fake_resolve(name, rdtype): + host = name if isinstance(name, str) else str(name) + host = host.rstrip(".").lower() + if rdtype == "CNAME": + if host == "start.example": + return [_Rdata("end.example.")] + raise Exception("nx") + if rdtype == "A": + if host == "end.example": + obj = SimpleNamespace(to_text=lambda: "203.0.113.10") + return [obj] + return [] + if rdtype == "AAAA": + return [] + if rdtype == "TXT": + raise Exception("no") + raise Exception("x") + + with patch("app.services.dns.dns.asyncresolver.Resolver") as resolver_cls: + inst = resolver_cls.return_value + inst.resolve = AsyncMock(side_effect=fake_resolve) + snap = await resolve_cname_chain("start.example") + assert snap.terminal_cname == "end.example" + assert snap.fingerprint == compute_fingerprint("end.example", ["203.0.113.10"])
@@ -0,0 +1,45 @@
+"""HTTP size limit and scheduler lock behaviour.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +import respx +from app.probes.http import probe_http +from app.scheduler import _locks, job_import_sources + + +@pytest.mark.asyncio +@respx.mock +async def test_http_response_size_limit(): + url = "http://tracker.example:80/announce" + big = b"x" * 2000 + + def _handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=big) + + respx.route(method="GET", host="tracker.example").mock(side_effect=_handler) + async with httpx.AsyncClient() as client: + outcome = await probe_http( + url, + client=client, + max_bytes=1000, + resolved_ips=["1.1.1.1"], + ) + assert outcome.response_valid is False + assert outcome.error_kind == "too_large" + + +@pytest.mark.asyncio +async def test_scheduler_skips_overlapping_import(): + # Hold the import lock and ensure job returns immediately without work + assert not _locks["import"].locked() + await _locks["import"].acquire() + try: + with patch("app.scheduler.import_all_sources", new_callable=AsyncMock) as mocked: + await job_import_sources() + mocked.assert_not_called() + finally: + _locks["import"].release()
@@ -0,0 +1,50 @@
+from __future__ import annotations + +from app.i18n import ( + encode_bootstrap_note, + from_accept_language, + render_bootstrap_note, + resolve_lang, + translate, +) + + +class _Req: + def __init__(self, *, cookie: str | None = None, accept: str | None = None): + self.cookies = {"lang": cookie} if cookie else {} + self.headers = {"accept-language": accept} if accept else {} + + +def test_cookie_wins_over_accept_language(): + req = _Req(cookie="pt", accept="en-US,en;q=0.9") + assert resolve_lang(req) == "pt" + + +def test_accept_language_en_without_cookie(): + req = _Req(accept="en-US,en;q=0.8") + assert resolve_lang(req) == "en" + + +def test_default_is_pt(): + assert resolve_lang(_Req()) == "pt" + assert from_accept_language(None) == "pt" + + +def test_bootstrap_note_follows_lang(): + raw = encode_bootstrap_note("empty_import", "boom") + pt = render_bootstrap_note("pt", raw) + en = render_bootstrap_note("en", raw) + assert "Inventário vazio" in pt + assert "boom" in pt + assert "Empty inventory" in en + assert "boom" in en + + +def test_legacy_bootstrap_note_passthrough(): + old = "Inventário vazio: alguma coisa" + assert render_bootstrap_note("en", old) == old + + +def test_translate_fallback(): + assert translate("en", "not_found") == "Tracker not found" + assert translate("pt", "not_found") == "Tracker não encontrado"
@@ -0,0 +1,112 @@
+"""Tests for the jokepool710 and XIU2 collectors and the shared feed runner.""" + +from __future__ import annotations + +import httpx +import pytest +import respx +from app.collectors.jokepool710 import Jokepool710Collector +from app.collectors.xiu2 import Xiu2Collector +from app.config import Settings +from app.db.models import Tracker, TrackerSource +from sqlalchemy import select + +FIXTURE_JOKEPOOL = """ +udp://tracker.example:6969/announce +http://tracker.example:80/announce +udp://tracker.example:6969/announce +""" + +FIXTURE_XIU2_ALL = """ +udp://a.example:6969/announce + +http://b.example:80/announce +""" + +FIXTURE_XIU2_BEST = """ +udp://a.example:6969/announce +""" + + +@pytest.mark.asyncio +@respx.mock +async def test_jokepool710_import_and_empty_does_not_wipe(session, settings: Settings): + settings = Settings( + APP_NAME="RastroTest", + DATABASE_URL=settings.database_url, + JOKEPOOL710_ALL_URL="https://example.test/joke.txt", + RUN_SCHEDULER=False, + ) + respx.get("https://example.test/joke.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_JOKEPOOL) + ) + collector = Jokepool710Collector() + async with httpx.AsyncClient() as client: + stats = await collector.import_once(session, client, settings) + assert stats.valid == 2 + assert stats.new == 2 + assert stats.error is None + + rows = (await session.execute(select(Tracker))).scalars().all() + assert len(rows) == 2 + sources = (await session.execute(select(TrackerSource))).scalars().all() + assert {s.source_name for s in sources} == {"jokepool710"} + + # Empty response must preserve existing data. + respx.get("https://example.test/joke.txt").mock(return_value=httpx.Response(200, text="\n\n")) + async with httpx.AsyncClient() as client: + stats2 = await collector.import_once(session, client, settings) + assert stats2.error is not None + rows2 = (await session.execute(select(Tracker))).scalars().all() + assert len(rows2) == 2 + + +@pytest.mark.asyncio +@respx.mock +async def test_xiu2_import_sets_best_flag(session, settings: Settings): + settings = Settings( + APP_NAME="RastroTest", + DATABASE_URL=settings.database_url, + XIU2_ALL_URL="https://example.test/xiu2-all.txt", + XIU2_BEST_URL="https://example.test/xiu2-best.txt", + RUN_SCHEDULER=False, + ) + respx.get("https://example.test/xiu2-all.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_XIU2_ALL) + ) + respx.get("https://example.test/xiu2-best.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_XIU2_BEST) + ) + collector = Xiu2Collector() + async with httpx.AsyncClient() as client: + stats = await collector.import_once(session, client, settings) + assert stats.valid == 2 + assert stats.new == 2 + assert stats.error is None + + sources = (await session.execute(select(TrackerSource))).scalars().all() + best = [s for s in sources if s.is_best_at_source] + assert len(best) == 1 + + +@pytest.mark.asyncio +@respx.mock +async def test_xiu2_best_failure_does_not_fail_import(session, settings: Settings): + settings = Settings( + APP_NAME="RastroTest", + DATABASE_URL=settings.database_url, + XIU2_ALL_URL="https://example.test/xiu2-all.txt", + XIU2_BEST_URL="https://example.test/xiu2-best.txt", + RUN_SCHEDULER=False, + ) + respx.get("https://example.test/xiu2-all.txt").mock( + return_value=httpx.Response(200, text=FIXTURE_XIU2_ALL) + ) + respx.get("https://example.test/xiu2-best.txt").mock(return_value=httpx.Response(503)) + collector = Xiu2Collector() + async with httpx.AsyncClient() as client: + stats = await collector.import_once(session, client, settings) + assert stats.error is None + assert stats.valid == 2 + sources = (await session.execute(select(TrackerSource))).scalars().all() + assert all(not s.is_best_at_source for s in sources)
@@ -0,0 +1,60 @@
+"""Normalization and SSRF unit tests.""" + +from __future__ import annotations + +import pytest +from app.services.normalize import ( + NormalizationError, + normalize_tracker_url, + parse_tracker_list_text, +) +from app.services.ssrf import assert_safe_destination, is_blocked_hostname, is_public_ip + + +def test_parse_blank_lines_and_duplicates(): + body = "udp://a.example:80/announce\n\n\nudp://a.example:80/announce\nhttp://b.example:80/announce\n" + items = parse_tracker_list_text(body) + assert items == [ + "udp://a.example:80/announce", + "http://b.example:80/announce", + ] + + +def test_normalize_basic_and_idna(): + n = normalize_tracker_url("HTTPS://EXEMPLO.COM:443/announce") + assert n.scheme == "https" + assert n.hostname == "exemplo.com" + assert n.port == 443 + assert n.canonical_url == "https://exemplo.com:443/announce" + + n2 = normalize_tracker_url("http://bücher.example/announce") + assert "xn--" in n2.hostname + assert n2.scheme == "http" + + +def test_reject_credentials_and_bad_port(): + with pytest.raises(NormalizationError): + normalize_tracker_url("http://user:pass@host.example:80/announce") + with pytest.raises(NormalizationError): + normalize_tracker_url("udp://host.example:99999/announce") + + +def test_unsupported_ws(): + n = normalize_tracker_url("wss://tracker.example/announce") + assert n.unsupported is True + + +def test_ssrf_blocks_private_and_metadata(): + assert is_public_ip("8.8.8.8") + assert not is_public_ip("127.0.0.1") + assert not is_public_ip("10.0.0.1") + assert not is_public_ip("192.168.1.1") + assert not is_public_ip("169.254.169.254") + assert not is_public_ip("::1") + assert not is_public_ip("fc00::1") + assert is_blocked_hostname("localhost") + assert is_blocked_hostname("metadata.google.internal") + with pytest.raises(ValueError): + assert_safe_destination("evil.example", ["127.0.0.1"]) + with pytest.raises(ValueError): + assert_safe_destination("localhost", ["8.8.8.8"])
@@ -0,0 +1,80 @@
+"""Weekly import cron trigger and import-staleness helper.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from app.config import Settings +from app.db.models import SourceImportRun +from app.db.queries import latest_successful_import_at +from app.scheduler import start_scheduler, stop_scheduler + + +@pytest.mark.asyncio +async def test_import_job_registered_as_weekly_cron(): + settings = Settings( + APP_NAME="RastroTest", + RUN_SCHEDULER=True, + IMPORT_DAY_OF_WEEK="sun", + IMPORT_HOUR_UTC=4, + ) + scheduler = start_scheduler(settings) + try: + job = scheduler.get_job("import_sources") + assert job is not None + trigger = str(job.trigger) + assert "day_of_week='sun'" in trigger + assert "hour='4'" in trigger + finally: + stop_scheduler() + + +@pytest.mark.asyncio +async def test_latest_successful_import_at(session): + now = datetime.now(UTC) + session.add_all( + [ + SourceImportRun( + source_name="a", + started_at=now - timedelta(days=10), + finished_at=now - timedelta(days=10), + status="ok", + ), + SourceImportRun( + source_name="b", + started_at=now - timedelta(days=2), + finished_at=now - timedelta(days=2), + status="not_modified", + ), + SourceImportRun( + source_name="c", + started_at=now, + finished_at=now, + status="failed", + ), + ] + ) + await session.commit() + + latest = await latest_successful_import_at(session) + assert latest is not None + if latest.tzinfo is None: + latest = latest.replace(tzinfo=UTC) + # The 2-day-old not_modified run wins; the failed "now" run does not count. + assert (now - latest) < timedelta(days=3) + + +@pytest.mark.asyncio +async def test_latest_successful_import_at_none_when_only_failures(session): + now = datetime.now(UTC) + session.add( + SourceImportRun( + source_name="a", + started_at=now, + finished_at=now, + status="failed", + ) + ) + await session.commit() + assert await latest_successful_import_at(session) is None
@@ -0,0 +1,106 @@
+"""DNS fingerprint helpers and scoring / smart list.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from app.db.models import ProbeResult, Tracker +from app.services.dns import compute_fingerprint +from app.services.scoring import compute_score, latency_score, metrics_from_results +from app.services.smart_list import SmartListEntry, SmartListFilters, format_smart_list_txt + + +def test_cname_fingerprint_deterministic(): + a = compute_fingerprint("cdn.example.net", ["1.2.3.4"]) + b = compute_fingerprint("cdn.example.net", ["9.9.9.9"]) + assert a == b + ips = compute_fingerprint(None, ["2.2.2.2", "1.1.1.1"]) + ips2 = compute_fingerprint(None, ["1.1.1.1", "2.2.2.2"]) + assert ips == ips2 + assert a != ips + + +def test_scoring_and_provisional(): + assert latency_score(100) == 20 + assert latency_score(250) == 16 + assert latency_score(2000) == 0 + assert ( + compute_score(uptime_7d=1.0, latency_median_7d=100, valid_rate_7d=1.0, measurement_count=2) + is None + ) + score = compute_score( + uptime_7d=1.0, latency_median_7d=100, valid_rate_7d=1.0, measurement_count=25 + ) + assert score == 100 + + tracker = Tracker( + canonical_url="udp://t.example:80/announce", + scheme="udp", + hostname="t.example", + port=80, + path="/announce", + current_status="up", + first_seen_at=datetime.now(UTC) - timedelta(days=3), + last_seen_at=datetime.now(UTC), + consecutive_failures=0, + ) + now = datetime.now(UTC) + results = [ + ProbeResult( + tracker_id=1, + checked_at=now - timedelta(hours=i), + status="up", + latency_ms=120, + response_valid=True, + ) + for i in range(5) + ] + m = metrics_from_results(tracker, results, now=now) + assert m.provisional is True + assert m.score is not None + assert m.uptime_7d == 1.0 + + +def test_smart_list_txt_blank_lines(): + t1 = Tracker( + id=1, + canonical_url="udp://a.example:80/announce", + scheme="udp", + hostname="a.example", + port=80, + path="/announce", + current_status="up", + first_seen_at=datetime.now(UTC), + last_seen_at=datetime.now(UTC), + consecutive_failures=0, + ) + t2 = Tracker( + id=2, + canonical_url="udp://b.example:80/announce", + scheme="udp", + hostname="b.example", + port=80, + path="/announce", + current_status="up", + first_seen_at=datetime.now(UTC), + last_seen_at=datetime.now(UTC), + consecutive_failures=0, + ) + from app.services.scoring import TrackerMetrics + + metrics = TrackerMetrics( + uptime_24h=1, + uptime_7d=1, + uptime_30d=1, + latency_median_7d=10, + latency_p95_7d=20, + valid_rate_7d=1, + measurement_count=20, + measurement_count_7d=20, + tracking_days=10, + score=90, + provisional=False, + ) + txt = format_smart_list_txt([SmartListEntry(t1, metrics), SmartListEntry(t2, metrics)]) + assert txt == "udp://a.example:80/announce\n\nudp://b.example:80/announce\n" + assert SmartListFilters().limit == 20
@@ -0,0 +1,71 @@
+"""Smart list diversity selection.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from app.db.models import ProbeResult, Tracker +from app.services.smart_list import SmartListFilters, build_smart_list +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _seed_tracker( + session: AsyncSession, + *, + url: str, + fp: str, + asn: int | None, + latency: float, +) -> Tracker: + now = datetime.now(UTC) + t = Tracker( + canonical_url=url, + scheme="udp", + hostname=url.split("://")[1].split(":")[0], + port=80, + path="/announce", + current_status="up", + first_seen_at=now - timedelta(days=30), + last_seen_at=now, + last_checked_at=now, + last_latency_ms=latency, + consecutive_failures=0, + infrastructure_fingerprint=fp, + asn=asn, + supports_ipv4=True, + supports_ipv6="not_tested", + ) + session.add(t) + await session.flush() + for i in range(12): + session.add( + ProbeResult( + tracker_id=t.id, + checked_at=now - timedelta(hours=i), + status="up", + latency_ms=latency, + response_valid=True, + ) + ) + await session.commit() + return t + + +@pytest.mark.asyncio +async def test_smart_list_diversity(session: AsyncSession): + await _seed_tracker(session, url="udp://a1.example:80/announce", fp="fp1", asn=100, latency=50) + await _seed_tracker(session, url="udp://a2.example:80/announce", fp="fp1", asn=100, latency=40) + await _seed_tracker(session, url="udp://b1.example:80/announce", fp="fp2", asn=100, latency=60) + await _seed_tracker(session, url="udp://c1.example:80/announce", fp="fp3", asn=200, latency=70) + + entries = await build_smart_list( + session, SmartListFilters(min_uptime=0.5, limit=20, diversity=True) + ) + urls = [e.tracker.canonical_url for e in entries] + # only one of fp1 + assert sum(1 for u in urls if u.startswith("udp://a")) == 1 + # asn 100 at most twice + asn100 = [e for e in entries if e.tracker.asn == 100] + assert len(asn100) <= 2 + assert "udp://c1.example:80/announce" in urls