url2pdf-tools/crawl_fast.js

// Fast concurrent PDF crawler - converts multiple pages in parallel
const puppeteer = require("puppeteer");
const fs = require("fs");
const path = require("path");
const { execSync } = require("child_process");

const BASE = process.argv[2];
const CONCURRENCY = parseInt(process.argv[3]) || 6;
if (!BASE) { console.error("Usage: node crawl_fast.js <url> [concurrency]"); process.exit(1); }

const OUT_DIR = "/tmp/doc_pages";
const FINAL = "/tmp/merged_guide.pdf";
const baseUrl = new URL(BASE);
const articleIdx = baseUrl.pathname.lastIndexOf("/");
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);

async function convertPage(browser, href, outPath, index, total) {
  if (fs.existsSync(outPath) && fs.statSync(outPath).size > 500) return true;
  const page = await browser.newPage();
  try {
    await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
    await page.pdf({ path: outPath, format: "A4", printBackground: true });
    await page.close();
    process.stdout.write(` [${index+1}/${total}]`);
    return true;
  } catch(e) {
    await page.close().catch(()=>{});
    process.stdout.write(` x`);
    fs.writeFileSync(outPath, ""); // marker for skip
    return false;
  }
}

(async () => {
  fs.rmSync(OUT_DIR, { recursive: true, force: true });
  fs.mkdirSync(OUT_DIR, { recursive: true });

  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
  const mainPage = await browser.newPage();

  console.log("🌐 Loading TOC:", BASE);
  await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
  await new Promise(r => setTimeout(r, 3000));

  // Extract all doc links
  const allLinks = await mainPage.evaluate(() => {
    const links = [];
    document.querySelectorAll("a[href]").forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
    });
    return links;
  });
  await mainPage.close();

  // Filter: same article dir, en-us only, unique filenames
  const seenFiles = new Set();
  const docPages = [];
  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (!u.pathname.startsWith(articleDir)) continue;
      if (!u.pathname.startsWith("/en-us/")) continue;
      if (u.hash) continue;
      const fn = u.pathname.split("/").pop();
      if (!fn || fn.length < 20) continue;
      
      // Skip obvious non-content pages
      const lower = link.text.toLowerCase();
      if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
      if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
      
      if (!seenFiles.has(fn)) {
        seenFiles.add(fn);
        docPages.push({ href: link.href, text: link.text });
      }
    } catch(e) {}
  }

  console.log(`📋 ${docPages.length} pages to convert with ${CONCURRENCY} concurrent tabs`);
  
  if (docPages.length === 0) { console.log("❌ No pages found"); process.exit(1); }

  const startTime = Date.now();
  const chunks = [];
  for (let i = 0; i < docPages.length; i += CONCURRENCY) {
    const batch = docPages.slice(i, i + CONCURRENCY);
    const tasks = batch.map((p, j) => {
      const out = path.join(OUT_DIR, String(i + j).padStart(4, "0") + ".pdf");
      chunks.push(out);
      return convertPage(browser, p.href, out, i + j, docPages.length);
    });
    await Promise.all(tasks);
  }

  await browser.close();

  const validPdfs = chunks.filter(f => fs.existsSync(f) && fs.statSync(f).size > 500);
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(0);
  console.log(`\n⏱ ${elapsed}s — ${validPdfs.length}/${docPages.length} pages converted`);

  if (validPdfs.length > 0) {
    console.log(`📚 Merging...`);
    execSync(`pdfunite ${validPdfs.join(" ")} ${FINAL}`, { stdio: "pipe" });
    const stats = fs.statSync(FINAL);
    console.log(`✅ ${FINAL} (${(stats.size/1024/1024).toFixed(1)}MB)`);
  }
})();