v1.0.0: URL-to-PDF + Doc Site Crawler

- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
2026-05-22 14:24:05 +08:00
commit 0d3e40f797
11 changed files with 1789 additions and 0 deletions
@@ -0,0 +1,120 @@
+// Batch crawl: process 100 pages, close/reopen browser to free memory
+const puppeteer = require("puppeteer");
+const fs = require("fs");
+
+const BASE = process.argv[2];
+const START = parseInt(process.argv[3]) || 0;
+const BATCH = parseInt(process.argv[4]) || 100;
+const OUT = "/tmp/apex_one_admin_guide.md";
+const URLS_FILE = "/tmp/apex_one_urls.json";
+const baseUrl = new URL(BASE);
+const articleIdx = baseUrl.pathname.lastIndexOf("/");
+const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
+
+async function getUrls() {
+  if (fs.existsSync(URLS_FILE)) {
+    return JSON.parse(fs.readFileSync(URLS_FILE, "utf8"));
+  }
+  
+  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
+  const page = await browser.newPage();
+  await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
+  await new Promise(r => setTimeout(r, 3000));
+
+  const allLinks = await page.evaluate(() => {
+    const links = [];
+    document.querySelectorAll("a[href]").forEach(a => {
+      const text = a.textContent.trim();
+      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
+    });
+    return links;
+  });
+  await browser.close();
+
+  const seenFiles = new Set();
+  const docPages = [];
+  for (const link of allLinks) {
+    try {
+      const u = new URL(link.href);
+      if (u.hostname !== baseUrl.hostname) continue;
+      if (!u.pathname.startsWith(articleDir)) continue;
+      if (!u.pathname.startsWith("/en-us/")) continue;
+      if (u.hash) continue;
+      const fn = u.pathname.split("/").pop();
+      if (!fn || fn.length < 20) continue;
+      const lower = link.text.toLowerCase();
+      if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
+      if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
+      if (!seenFiles.has(fn)) {
+        seenFiles.add(fn);
+        docPages.push({ href: link.href, text: link.text });
+      }
+    } catch(e) {}
+  }
+  fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
+  return docPages;
+}
+
+async function extractBatch(urls, start, count) {
+  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
+  const end = Math.min(start + count, urls.length);
+  
+  for (let i = start; i < end; i++) {
+    const { href, text } = urls[i];
+    try {
+      const page = await browser.newPage();
+      await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
+      await new Promise(r => setTimeout(r, 300));
+
+      const content = await page.evaluate(() => {
+        const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
+        let main = document.body;
+        for (const sel of mainSel) {
+          const el = document.querySelector(sel);
+          if (el) { main = el; break; }
+        }
+        const h1 = main.querySelector("h1, h2");
+        const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim();
+        
+        const blocks = [];
+        main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
+          const txt = el.textContent.trim();
+          if (!txt || txt.length < 3) return;
+          const tag = el.tagName;
+          if (tag.match(/^H[1-4]$/)) {
+            blocks.push("\n" + "#".repeat(parseInt(tag[1]) + 1) + " " + txt + "\n");
+          } else if (tag === "LI") {
+            blocks.push("- " + txt);
+          } else if (tag === "PRE") {
+            blocks.push("\n```\n" + txt + "\n```\n");
+          } else {
+            blocks.push(txt);
+          }
+        });
+        return { title, body: blocks.join("\n\n") };
+      });
+
+      await page.close();
+
+      const out = `## ${i + 1}. ${content.title}\n\n${content.body || "_no content_"}  \n`;
+      fs.appendFileSync(OUT, out);
+    } catch(e) {
+      fs.appendFileSync(OUT, `## ${i + 1}. ${text}\n\n_Error_  \n`);
+    }
+  }
+  
+  await browser.close();
+  console.log(`✅ [${start+1}-${end}/${urls.length}]`);
+}
+
+(async () => {
+  const urls = await getUrls();
+  console.log(`📋 ${urls.length} pages total, starting at ${START}, batch ${BATCH}`);
+  
+  // Write header if starting from 0
+  if (START === 0) {
+    fs.writeFileSync(OUT, `# Apex One Service Pack 1 (2025) — Admin Guide\n\n> ${urls.length} pages from ${BASE}\n\n`);
+  }
+  
+  await extractBatch(urls, START, BATCH);
+})();