app/services/smart_list.py (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
from __future__ import annotations
from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.db.models import ProbeResult, Tracker
from app.services.scoring import TrackerMetrics, load_metrics
@dataclass
class SmartListFilters:
protocol: str | None = None # udp|http|https
min_uptime: float = 0.95
max_latency_ms: float | None = None
ip_version: str = "any" # any|ipv4|ipv6
limit: int = 20
min_age_days: float = 0.0
diversity: bool = True
require_valid: bool = True
min_measurements: int = 10
@dataclass
class SmartListEntry:
tracker: Tracker
metrics: TrackerMetrics
async def build_smart_list(
session: AsyncSession, filters: SmartListFilters | None = None
) -> list[SmartListEntry]:
filters = filters or SmartListFilters()
limit = max(1, min(50, filters.limit))
stmt = (
select(Tracker).where(Tracker.current_status == "up").options(selectinload(Tracker.sources))
)
if filters.protocol:
stmt = stmt.where(Tracker.scheme == filters.protocol.lower())
result = await session.execute(stmt)
trackers = list(result.scalars().unique().all())
scored: list[SmartListEntry] = []
for tracker in trackers:
metrics = await load_metrics(session, tracker)
if metrics.measurement_count < filters.min_measurements:
continue
if filters.require_valid and (metrics.valid_rate_7d or 0) <= 0:
# require at least some valid responses; last probe should be valid
last = await session.execute(
select(ProbeResult)
.where(ProbeResult.tracker_id == tracker.id)
.order_by(ProbeResult.checked_at.desc())
.limit(1)
)
last_row = last.scalar_one_or_none()
if last_row is None or not last_row.response_valid:
continue
if metrics.uptime_7d is None or metrics.uptime_7d < filters.min_uptime:
continue
if filters.max_latency_ms is not None and (
metrics.latency_median_7d is None or metrics.latency_median_7d > filters.max_latency_ms
):
continue
if metrics.tracking_days < filters.min_age_days:
continue
if filters.ip_version == "ipv4" and not tracker.supports_ipv4:
continue
if filters.ip_version == "ipv6" and tracker.supports_ipv6 != "true":
continue
scored.append(SmartListEntry(tracker=tracker, metrics=metrics))
scored.sort(
key=lambda e: (
-(e.metrics.score or 0),
e.metrics.latency_median_7d if e.metrics.latency_median_7d is not None else 1e9,
)
)
if not filters.diversity:
return scored[:limit]
selected: list[SmartListEntry] = []
seen_fps: set[str] = set()
asn_counts: dict[int, int] = {}
for entry in scored:
fp = entry.tracker.infrastructure_fingerprint
if fp and fp in seen_fps:
continue
asn = entry.tracker.asn
if asn is not None and asn_counts.get(asn, 0) >= 2:
continue
selected.append(entry)
if fp:
seen_fps.add(fp)
if asn is not None:
asn_counts[asn] = asn_counts.get(asn, 0) + 1
if len(selected) >= limit:
break
return selected
def format_smart_list_txt(entries: list[SmartListEntry]) -> str:
if not entries:
return ""
return "\n\n".join(e.tracker.canonical_url for e in entries) + "\n"
def format_smart_list_json(entries: list[SmartListEntry]) -> list[dict]:
out = []
for e in entries:
out.append(
{
"id": e.tracker.id,
"url": e.tracker.canonical_url,
"scheme": e.tracker.scheme,
"score": e.metrics.score,
"uptime_7d": e.metrics.uptime_7d,
"latency_median_7d": e.metrics.latency_median_7d,
"infrastructure_fingerprint": e.tracker.infrastructure_fingerprint,
"asn": e.tracker.asn,
}
)
return out
async def build_live_list(session: AsyncSession) -> list[str]:
result = await session.execute(
select(Tracker)
.where(Tracker.current_status == "up")
.order_by(Tracker.last_latency_ms.asc().nullslast())
)
return [t.canonical_url for t in result.scalars().all()]
|