fetch.py (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
import re
import asyncio
import hashlib
from datetime import datetime, timezone
from typing import Optional, List, Dict, Any
import httpx
import feedparser
from config import HTTP_TIMEOUT, HTTP_HEADERS
def clean_summary(raw: str, limit: int = 300) -> str:
text = re.sub(r"<[^>]+>", "", raw or "").strip()
return text[:limit] + ("…" if len(text) > limit else "")
def parse_date(struct_time) -> Optional[datetime]:
if not struct_time:
return None
return datetime(*struct_time[:6], tzinfo=timezone.utc)
def _guid(entry: Any, link: str, title: str, published: Optional[datetime]) -> str:
raw = entry.get("id") or entry.get("guid") or link
if raw:
return raw
base = f"{title}|{published.isoformat() if published else ''}"
return "sha1:" + hashlib.sha1(base.encode("utf-8")).hexdigest()
async def fetch_feed(
client: httpx.AsyncClient, feed: Dict[str, str]
) -> List[Dict[str, Any]]:
try:
resp = await client.get(feed["url"], headers=HTTP_HEADERS,
timeout=HTTP_TIMEOUT, follow_redirects=True)
resp.raise_for_status()
except (httpx.HTTPError, httpx.TimeoutException):
return []
parsed = feedparser.parse(resp.content)
items: List[Dict[str, Any]] = []
for entry in parsed.entries:
link = entry.get("link", "")
title = entry.get("title", "(sem título)")
published = parse_date(
entry.get("published_parsed") or entry.get("updated_parsed")
)
items.append({
"feed_id": feed["id"],
"source": feed["name"],
"category": feed.get("category", "geral"),
"guid": _guid(entry, link, title, published),
"title": title,
"link": link,
"summary": clean_summary(entry.get("summary", "")),
"published": published,
})
return items
async def fetch_all(feeds: List[Dict[str, str]]) -> List[Dict[str, Any]]:
async with httpx.AsyncClient() as client:
results = await asyncio.gather(*[fetch_feed(client, f) for f in feeds])
return [item for sublist in results for item in sublist]
|