all repos — mcp-news @ 20021af083021d16f6f1732dc3fa370e756ecfc5

personal mcp server for news

collector.py (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
import asyncio
import logging

import config
import db
from feeds import load_feeds
from fetch import fetch_all

log = logging.getLogger("news_mcp.collector")


async def poll_once() -> int:
    feeds = load_feeds()
    articles = await fetch_all(feeds)
    inserted = await asyncio.to_thread(db.upsert_articles, articles)
    removed = await asyncio.to_thread(db.prune, config.RETENTION_DAYS)
    log.info("coleta: %d feeds, %d artigos baixados, %d novos, %d podados",
             len(feeds), len(articles), inserted, removed)
    return inserted


async def run_collector() -> None:
    while True:
        try:
            await poll_once()
        except Exception:
            log.exception("falha na coleta; tentando de novo no próximo ciclo")
        await asyncio.sleep(config.POLL_INTERVAL_MIN * 60)