all repos — red-anatomia @ 43cc40161c75c7c9bb5bef2c1ce2d551b7f24484

red-anatomia

src/collectors/http.ts (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 
import got, { type Response } from 'got';
import type { Observations, Redirecionamento, CookieObservado } from '../core/types.js';

const USER_AGENT = 'Mozilla/5.0 (compatible; Red; +https://github.com/pablomurad/red)';

export interface RespostaHttp {
  resposta: Response<string>;
  redirecionamentos: Redirecionamento[];
}

export async function coletarHttp(
  urlSolicitada: string,
  obs: Observations,
): Promise<RespostaHttp | undefined> {
  const redirecionamentos: Redirecionamento[] = [];

  try {
    const resposta = await got(urlSolicitada, {
      method: 'GET',
      throwHttpErrors: false,
      followRedirect: true,
      maxRedirects: 10,
      timeout: { request: 15000 },
      headers: {
        'user-agent': USER_AGENT,
        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'accept-language': 'pt-BR,pt;q=0.9,en;q=0.8',
      },
      hooks: {
        beforeRedirect: [
          (options, resp) => {
            redirecionamentos.push({
              de: resp.url,
              para: options.url?.toString() ?? '',
              status: resp.statusCode,
            });
          },
        ],
      },
    });

    obs.urlFinal = resposta.url;
    obs.statusFinal = resposta.statusCode;
    obs.redirecionamentos = redirecionamentos;
    obs.html = typeof resposta.body === 'string' ? resposta.body : '';
    obs.headers = normalizarHeaders(resposta.headers);
    obs.cookies = extrairCookies(resposta.headers['set-cookie']);
    inferirServidorECdn(obs);

    return { resposta, redirecionamentos };
  } catch (erro) {
    obs.avisos.push(`Falha ao buscar a URL: ${(erro as Error).message}`);
    return undefined;
  }
}

function normalizarHeaders(headers: Record<string, unknown>): Record<string, string> {
  const saida: Record<string, string> = {};
  for (const [chave, valor] of Object.entries(headers)) {
    if (valor == null) continue;
    saida[chave.toLowerCase()] = Array.isArray(valor) ? valor.join(', ') : String(valor);
  }
  return saida;
}

function extrairCookies(setCookie: string[] | undefined): CookieObservado[] {
  if (!setCookie) return [];
  return setCookie.map((linha) => {
    const [par, ...resto] = linha.split(';');
    const nome = (par ?? '').split('=')[0]?.trim() ?? '';
    return { nome, atributos: resto.map((a) => a.trim()).join('; ') || undefined };
  });
}

function inferirServidorECdn(obs: Observations): void {
  const h = obs.headers;
  if (h['server']) obs.servidorAparente = h['server'];

  if (h['cf-ray'] || /cloudflare/i.test(h['server'] ?? '')) obs.cdnAparente = 'Cloudflare';
  else if (h['x-vercel-id'] || /vercel/i.test(h['server'] ?? '')) obs.cdnAparente = 'Vercel';
  else if (h['x-amz-cf-id'] || /cloudfront/i.test(h['via'] ?? '')) obs.cdnAparente = 'Amazon CloudFront';
  else if (h['x-fastly-request-id'] || /fastly/i.test(h['via'] ?? '')) obs.cdnAparente = 'Fastly';
  else if (h['x-akamai-transformed'] || /akamai/i.test(h['server'] ?? '')) obs.cdnAparente = 'Akamai';
  else if (/netlify/i.test(h['server'] ?? '') || h['x-nf-request-id']) obs.cdnAparente = 'Netlify';
}