url2pdf-tools/crawl.js

// Crawl a Trend Micro documentation site (or similar JS doc sites)
// Deduplicates language variants, filters out nav/UI links
const puppeteer = require("puppeteer");
const fs = require("fs");
const path = require("path");
const { execSync } = require("child_process");

const BASE = process.argv[2];
if (!BASE) { console.error("Usage: node crawl.js <doc-page-url>"); process.exit(1); }

const OUT_DIR = "/tmp/doc_pages";
const FINAL = "/tmp/merged_guide.pdf";
const baseUrl = new URL(BASE);

// Extract the documentation set prefix from URL
// e.g. /en-us/documentation/article/apex-one-service-pack-1-2025-server-online-help-
const articleIdx = baseUrl.pathname.lastIndexOf("/");
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);

(async () => {
  fs.rmSync(OUT_DIR, { recursive: true, force: true });
  fs.mkdirSync(OUT_DIR, { recursive: true });

  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
  const page = await browser.newPage();

  console.log("🌐 Loading:", BASE);
  await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
  await new Promise(r => setTimeout(r, 3000));

  // Extract ALL links from the fully loaded page
  const allLinks = await page.evaluate(() => {
    const links = [];
    document.querySelectorAll("a[href]").forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) {
        links.push({ href: a.href, text: text.slice(0, 80) });
      }
    });
    return links;
  });

  // Filter: same host, under same article directory, not a hash link
  // Also: unique by page filename (dedupe language variants)
  const seenFiles = new Set();
  const docPages = [];

  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (u.hash) continue;

      const pathname = u.pathname;
      // Must be under the same article directory
      if (!pathname.startsWith(articleDir)) continue;

      // Get the page filename
      const filename = pathname.split("/").pop();
      // Must match the doc page pattern (long hyphenated names ending with article ID)
      if (!filename || filename.length < 20) continue;

      // Skip language-switch duplicates (same filename, different language path)
      // The paths are like: /en-us/..., /fr-fr/..., /de-de/..., etc.
      // Filter to only English (en-us) pages
      if (!pathname.startsWith("/en-us/")) continue;

      if (!seenFiles.has(filename)) {
        seenFiles.add(filename);
        docPages.push({ href: link.href, text: link.text });
      }
    } catch (e) {}
  }

  console.log(`📋 Found ${docPages.length} unique English doc pages`);
  if (docPages.length === 0) {
    console.log("⚠️ No pages found with strict filter. Trying broader filter...");
    // Retry with just articleDir prefix, no language filter
    for (const link of allLinks) {
      try {
        const u = new URL(link.href);
        if (u.hostname !== baseUrl.hostname) continue;
        if (u.hash) continue;
        if (!u.pathname.startsWith(articleDir)) continue;
        const filename = u.pathname.split("/").pop();
        if (!filename || filename.length < 20) continue;
        if (!seenFiles.has(filename)) {
          seenFiles.add(filename);
          docPages.push({ href: link.href, text: link.text });
        }
      } catch (e) {}
    }
    console.log(`📋 Found ${docPages.length} pages with broader filter`);
  }

  if (docPages.length === 0) {
    console.log("❌ Still no doc pages. Dumping first 30 link samples:");
    allLinks.slice(0, 30).forEach(l => console.log("  ", l.href));
    process.exit(1);
  }

  // Print a few
  docPages.slice(0, 5).forEach((p, i) => console.log(`  ${i + 1}. ${p.href.split("/").pop().slice(0, 50)}`));
  console.log(`  ... and ${docPages.length - Math.min(5, docPages.length)} more`);

  // Convert each page to PDF
  const pdfs = [];
  for (let i = 0; i < docPages.length; i++) {
    const { href, text } = docPages[i];
    const fname = path.join(OUT_DIR, String(i).padStart(4, "0") + ".pdf");
    
    if (fs.existsSync(fname) && fs.statSync(fname).size > 1000) {
      console.log(`  ⏭ [${i+1}/${docPages.length}] skip: ${text.slice(0, 60)}`);
      pdfs.push(fname);
      continue;
    }
    
    console.log(`  📄 [${i+1}/${docPages.length}] ${text.slice(0, 60)}`);
    try {
      const p = await browser.newPage();
      await p.goto(href, { waitUntil: "networkidle2", timeout: 30000 });
      // Short wait for dynamic content
      await new Promise(r => setTimeout(r, 1000));
      await p.pdf({ path: fname, format: "A4", printBackground: true });
      await p.close();
      pdfs.push(fname);
    } catch (e) {
      console.log(`    ⚠️ Failed: ${e.message.slice(0, 80)}`);
    }
  }

  await browser.close();

  if (pdfs.length > 0) {
    console.log(`\n📚 Merging ${pdfs.length} PDFs...`);
    execSync(`pdfunite ${pdfs.join(" ")} ${FINAL}`, { stdio: "pipe" });
    const stats = fs.statSync(FINAL);
    console.log(`✅ Done! ${FINAL} (${(stats.size/1024).toFixed(0)}KB, ${pdfs.length} pages)`);
  } else {
    console.log("❌ No PDFs generated");
  }
})();