url2pdf-crawler-local/crawl_hpe.js

// Crawl HPE Aruba Networking AOS 10.x Release Notes
// Uses puppeteer-core + existing Chrome for Testing
const puppeteer = require('puppeteer-core');
const fs = require('fs');

const CHROME_PATH = '/Users/admin/.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing';
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
const URLS_FILE = OUT_DIR + '/urls.json';
const MD_FILE = OUT_DIR + '/all-releases.md';

if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });

const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
const EXTRA_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
  'Accept-Language': 'en-US,en;q=0.9',
  'Accept-Encoding': 'gzip, deflate, br'
};

async function setupPage(browser) {
  const page = await browser.newPage();
  await page.setUserAgent(USER_AGENT);
  await page.setExtraHTTPHeaders(EXTRA_HEADERS);
  return page;
}

async function getAllUrls() {
  if (fs.existsSync(URLS_FILE)) {
    const urls = JSON.parse(fs.readFileSync(URLS_FILE, 'utf8'));
    console.log(`📋 ${urls.length} URLs (cached)`);
    return urls;
  }

  const browser = await puppeteer.launch({
    executablePath: CHROME_PATH,
    headless: true,
    args: CHROME_ARGS
  });
  const page = await setupPage(browser);
  await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
  await new Promise(r => setTimeout(r, 3000));

  const baseUrl = new URL(BASE);
  const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);

  const allLinks = await page.evaluate(() => {
    const links = [];
    document.querySelectorAll('a[href]').forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
    });
    return links;
  });

  await browser.close();

  const seen = new Set();
  const docPages = [];
  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (!u.pathname.startsWith(articleDir)) continue;
      if (u.hash) continue;
      if (/javascript:|^#|^\?/i.test(link.href)) continue;
      const fn = u.pathname.split('/').pop();
      if (!fn || fn.length < 3) continue;
      if (!seen.has(fn)) {
        seen.add(fn);
        docPages.push({ href: link.href, text: link.text });
      }
    } catch (e) {}
  }

  docPages.unshift({ href: BASE, text: 'All Releases (Index)' });
  fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
  console.log(`📋 ${docPages.length} URLs found`);
  return docPages;
}

async function extractPage(page, href) {
  await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
  await new Promise(r => setTimeout(r, 600));

  return await page.evaluate(() => {
    const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.MCWebHelpFramesetLink'];
    let main = document.body;
    for (const sel of mainSel) {
      const el = document.querySelector(sel);
      if (el) { main = el; break; }
    }
    const h1 = main.querySelector('h1, h2');
    const title = h1 ? h1.textContent.trim() : document.title.trim();

    const blocks = [];
    main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
      const txt = el.textContent.trim();
      if (!txt || txt.length < 3) return;
      const tag = el.tagName;
      if (tag.match(/^H[1-4]$/)) {
        blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
      } else if (tag === 'LI') {
        blocks.push('- ' + txt);
      } else if (tag === 'PRE' || tag === 'CODE') {
        blocks.push('\n```\n' + txt + '\n```\n');
      } else if (tag === 'TABLE') {
        const rows = [];
        el.querySelectorAll('tr').forEach(tr => {
          const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
          if (cells) rows.push('| ' + cells + ' |');
        });
        if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
      } else {
        blocks.push(txt);
      }
    });
    return { title, body: blocks.join('\n\n') };
  });
}

async function crawl() {
  const urls = await getAllUrls();
  console.log(`\n📥 Starting crawl of ${urls.length} pages...\n`);

  fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes\n\n> Source: ${BASE}\n> Pages: ${urls.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);

  const browser = await puppeteer.launch({
    executablePath: CHROME_PATH,
    headless: true,
    args: CHROME_ARGS
  });
  const page = await setupPage(browser);

  let ok = 0, fail = 0;
  for (let i = 0; i < urls.length; i++) {
    const { href, text } = urls[i];
    try {
      const content = await extractPage(page, href);
      const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      ok++;
      console.log(`✅ [${i+1}/${urls.length}] ${content.title.slice(0, 60).replace(/\n/g, ' ')}`);
    } catch (e) {
      fail++;
      const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      console.log(`❌ [${i+1}/${urls.length}] ${e.message.slice(0, 80)}`);
    }
  }

  await browser.close();
  console.log(`\n📊 Done: ${ok} OK, ${fail} failed`);
  console.log(`📁 Output: ${MD_FILE}`);
}

crawl().catch(err => { console.error(err); process.exit(1); });