all repos — red-anatomia @ main

red-anatomia

src/collectors/robots.ts (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 
import got from 'got';
import type { Observations } from '../core/types.js';

export async function coletarRobots(obs: Observations): Promise<void> {
  let base: URL;
  try {
    base = new URL(obs.urlFinal || obs.urlSolicitada);
  } catch {
    return;
  }

  obs.robotsTxt = await buscarTexto(new URL('/robots.txt', base).toString());

  const declarado = obs.robotsTxt?.match(/^\s*sitemap:\s*(\S+)/im)?.[1];
  const urlSitemap = declarado ?? new URL('/sitemap.xml', base).toString();
  obs.sitemapXml = await buscarTexto(urlSitemap, 200_000);
}

async function buscarTexto(url: string, limite = 100_000): Promise<string | undefined> {
  try {
    const resp = await got(url, {
      throwHttpErrors: false,
      timeout: { request: 8000 },
      headers: { 'user-agent': 'Red' },
    });
    if (resp.statusCode >= 200 && resp.statusCode < 300 && typeof resp.body === 'string') {
      return resp.body.slice(0, limite);
    }
  } catch {
    return undefined;
  }
  return undefined;
}