v1.0.0: URL-to-PDF + Doc Site Crawler

- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
2026-05-22 14:24:05 +08:00
commit 0d3e40f797
11 changed files with 1789 additions and 0 deletions
@@ -0,0 +1,142 @@
+// Crawl a Trend Micro documentation site (or similar JS doc sites)
+// Deduplicates language variants, filters out nav/UI links
+const puppeteer = require("puppeteer");
+const fs = require("fs");
+const path = require("path");
+const { execSync } = require("child_process");
+
+const BASE = process.argv[2];
+if (!BASE) { console.error("Usage: node crawl.js <doc-page-url>"); process.exit(1); }
+
+const OUT_DIR = "/tmp/doc_pages";
+const FINAL = "/tmp/merged_guide.pdf";
+const baseUrl = new URL(BASE);
+
+// Extract the documentation set prefix from URL
+// e.g. /en-us/documentation/article/apex-one-service-pack-1-2025-server-online-help-
+const articleIdx = baseUrl.pathname.lastIndexOf("/");
+const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
+
+(async () => {
+  fs.rmSync(OUT_DIR, { recursive: true, force: true });
+  fs.mkdirSync(OUT_DIR, { recursive: true });
+
+  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
+  const page = await browser.newPage();
+
+  console.log("🌐 Loading:", BASE);
+  await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
+  await new Promise(r => setTimeout(r, 3000));
+
+  // Extract ALL links from the fully loaded page
+  const allLinks = await page.evaluate(() => {
+    const links = [];
+    document.querySelectorAll("a[href]").forEach(a => {
+      const text = a.textContent.trim();
+      if (text.length > 2) {
+        links.push({ href: a.href, text: text.slice(0, 80) });
+      }
+    });
+    return links;
+  });
+
+  // Filter: same host, under same article directory, not a hash link
+  // Also: unique by page filename (dedupe language variants)
+  const seenFiles = new Set();
+  const docPages = [];
+
+  for (const link of allLinks) {
+    try {
+      const u = new URL(link.href);
+      if (u.hostname !== baseUrl.hostname) continue;
+      if (u.hash) continue;
+
+      const pathname = u.pathname;
+      // Must be under the same article directory
+      if (!pathname.startsWith(articleDir)) continue;
+
+      // Get the page filename
+      const filename = pathname.split("/").pop();
+      // Must match the doc page pattern (long hyphenated names ending with article ID)
+      if (!filename || filename.length < 20) continue;
+
+      // Skip language-switch duplicates (same filename, different language path)
+      // The paths are like: /en-us/..., /fr-fr/..., /de-de/..., etc.
+      // Filter to only English (en-us) pages
+      if (!pathname.startsWith("/en-us/")) continue;
+
+      if (!seenFiles.has(filename)) {
+        seenFiles.add(filename);
+        docPages.push({ href: link.href, text: link.text });
+      }
+    } catch (e) {}
+  }
+
+  console.log(`📋 Found ${docPages.length} unique English doc pages`);
+  if (docPages.length === 0) {
+    console.log("⚠️ No pages found with strict filter. Trying broader filter...");
+    // Retry with just articleDir prefix, no language filter
+    for (const link of allLinks) {
+      try {
+        const u = new URL(link.href);
+        if (u.hostname !== baseUrl.hostname) continue;
+        if (u.hash) continue;
+        if (!u.pathname.startsWith(articleDir)) continue;
+        const filename = u.pathname.split("/").pop();
+        if (!filename || filename.length < 20) continue;
+        if (!seenFiles.has(filename)) {
+          seenFiles.add(filename);
+          docPages.push({ href: link.href, text: link.text });
+        }
+      } catch (e) {}
+    }
+    console.log(`📋 Found ${docPages.length} pages with broader filter`);
+  }
+
+  if (docPages.length === 0) {
+    console.log("❌ Still no doc pages. Dumping first 30 link samples:");
+    allLinks.slice(0, 30).forEach(l => console.log("  ", l.href));
+    process.exit(1);
+  }
+
+  // Print a few
+  docPages.slice(0, 5).forEach((p, i) => console.log(`  ${i + 1}. ${p.href.split("/").pop().slice(0, 50)}`));
+  console.log(`  ... and ${docPages.length - Math.min(5, docPages.length)} more`);
+
+  // Convert each page to PDF
+  const pdfs = [];
+  for (let i = 0; i < docPages.length; i++) {
+    const { href, text } = docPages[i];
+    const fname = path.join(OUT_DIR, String(i).padStart(4, "0") + ".pdf");
+    
+    if (fs.existsSync(fname) && fs.statSync(fname).size > 1000) {
+      console.log(`  ⏭ [${i+1}/${docPages.length}] skip: ${text.slice(0, 60)}`);
+      pdfs.push(fname);
+      continue;
+    }
+    
+    console.log(`  📄 [${i+1}/${docPages.length}] ${text.slice(0, 60)}`);
+    try {
+      const p = await browser.newPage();
+      await p.goto(href, { waitUntil: "networkidle2", timeout: 30000 });
+      // Short wait for dynamic content
+      await new Promise(r => setTimeout(r, 1000));
+      await p.pdf({ path: fname, format: "A4", printBackground: true });
+      await p.close();
+      pdfs.push(fname);
+    } catch (e) {
+      console.log(`    ⚠️ Failed: ${e.message.slice(0, 80)}`);
+    }
+  }
+
+  await browser.close();
+
+  if (pdfs.length > 0) {
+    console.log(`\n📚 Merging ${pdfs.length} PDFs...`);
+    execSync(`pdfunite ${pdfs.join(" ")} ${FINAL}`, { stdio: "pipe" });
+    const stats = fs.statSync(FINAL);
+    console.log(`✅ Done! ${FINAL} (${(stats.size/1024).toFixed(0)}KB, ${pdfs.length} pages)`);
+  } else {
+    console.log("❌ No PDFs generated");
+  }
+})();