v1.0.0: URL-to-PDF + Doc Site Crawler

- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
2026-05-22 14:24:05 +08:00
commit 0d3e40f797
11 changed files with 1789 additions and 0 deletions
@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import subprocess, tempfile, os, json, urllib.parse, time, threading, shutil
+
+JOBS_DIR = "/tmp/url2pdf_jobs"
+os.makedirs(JOBS_DIR, exist_ok=True)
+CWD = "/opt/url2pdf"
+
+PUPPETEER_JS = """
+const puppeteer = require("puppeteer");
+(async () => {
+  const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]});
+  const page = await browser.newPage();
+  await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000});
+  await page.pdf({path: process.argv[3], format: "A4", printBackground: true});
+  await browser.close();
+})();
+"""
+
+CRAWLER_JS = """
+// Documentation crawler - cwd must have puppeteer installed
+const puppeteer = require("puppeteer");
+const fs = require("fs");
+const path = require("path");
+const { execSync } = require("child_process");
+
+const BASE = process.argv[2];
+const JOB_DIR = process.argv[3];
+const baseUrl = new URL(BASE);
+const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf("/") + 1);
+
+function log(msg) {
+  fs.appendFileSync(path.join(JOB_DIR, "log.txt"), msg + "\\n");
+  process.stdout.write(msg + "\\n");
+}
+
+(async () => {
+  const statusPath = path.join(JOB_DIR, "status.json");
+  function setStatus(s) {
+    fs.writeFileSync(statusPath, JSON.stringify({status: s, time: new Date().toISOString()}));
+  }
+  
+  try {
+    setStatus("extracting_urls");
+    const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
+    const mainPage = await browser.newPage();
+    
+    await mainPage.goto(BASE, {waitUntil: "networkidle2", timeout: 30000});
+    await new Promise(r => setTimeout(r, 3000));
+    
+    const allLinks = await mainPage.evaluate(() => {
+      const links = [];
+      document.querySelectorAll("a[href]").forEach(a => {
+        const text = a.textContent.trim();
+        if (text.length > 2) links.push({href: a.href, text: text.slice(0, 80)});
+      });
+      return links;
+    });
+    await mainPage.close();
+    await browser.close();
+
+    // Filter unique doc pages (en-us only, dedupe by filename, skip language switchers)
+    const seenFiles = new Set();
+    const docPages = [];
+    for (const link of allLinks) {
+      try {
+        const u = new URL(link.href);
+        if (u.hostname !== baseUrl.hostname) continue;
+        if (!u.pathname.startsWith(articleDir)) continue;
+        if (!u.pathname.startsWith("/en-us/")) continue;
+        if (u.hash) continue;
+        const fn = u.pathname.split("/").pop();
+        if (!fn || fn.length < 20) continue;
+        const lower = link.text.toLowerCase();
+        if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
+        if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
+        if (!seenFiles.has(fn)) {
+          seenFiles.add(fn);
+          docPages.push({href: link.href, text: link.text});
+        }
+      } catch(e) {}
+    }
+
+    log("Found " + docPages.length + " pages");
+    setStatus("crawling:" + docPages.length);
+    
+    // Process in batches of 20 to avoid OOM
+    const BATCH = 20;
+    const mdPath = path.join(JOB_DIR, "output.md");
+    const pdfDir = path.join(JOB_DIR, "pdfs");
+    fs.mkdirSync(pdfDir, {recursive: true});
+    
+    // Write MD header
+    fs.writeFileSync(mdPath, "# " + baseUrl.hostname.split(".")[0] + " Documentation\\n\\n> " + docPages.length + " pages from " + BASE + "\\n\\n");
+    
+    const allPdfs = [];
+    
+    for (let batchStart = 0; batchStart < docPages.length; batchStart += BATCH) {
+      const batchEnd = Math.min(batchStart + BATCH, docPages.length);
+      setStatus("processing:" + batchStart + "/" + docPages.length);
+      
+      const b = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
+      
+      for (let i = batchStart; i < batchEnd; i++) {
+        const {href, text} = docPages[i];
+        try {
+          const p = await b.newPage();
+          await p.goto(href, {waitUntil: "networkidle2", timeout: 20000});
+          await new Promise(r => setTimeout(r, 300));
+          
+          // Extract text for MD
+          const content = await p.evaluate(() => {
+            const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
+            let main = document.body;
+            for (const sel of mainSel) { const el = document.querySelector(sel); if (el) { main = el; break; } }
+            const h1 = main.querySelector("h1, h2");
+            const title = h1 ? h1.textContent.trim() : document.title.replace(/ \\| .*$/, "").trim();
+            const blocks = [];
+            main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
+              const txt = el.textContent.trim();
+              if (!txt || txt.length < 3) return;
+              const tag = el.tagName;
+              if (tag.match(/^H[1-4]$/)) blocks.push("\\n" + "#".repeat(parseInt(tag[1])+1) + " " + txt + "\\n");
+              else if (tag === "LI") blocks.push("- " + txt);
+              else if (tag === "PRE") blocks.push("\\n```\\n" + txt + "\\n```\\n");
+              else blocks.push(txt);
+            });
+            return {title, body: blocks.join("\\n\\n")};
+          });
+          
+          fs.appendFileSync(mdPath, "## " + (i+1) + ". " + content.title + "\\n\\n" + (content.body || "_") + "\\n\\n");
+          
+          // PDF for this page
+          const pdfFile = path.join(pdfDir, String(i).padStart(4, "0") + ".pdf");
+          await p.pdf({path: pdfFile, format: "A4", printBackground: true});
+          allPdfs.push(pdfFile);
+          await p.close();
+        } catch(e) {
+          fs.appendFileSync(mdPath, "## " + (i+1) + ". " + text + "\\n\\n_Error_\\n\\n");
+        }
+      }
+      
+      await b.close();
+      log("Batch " + (batchStart/BATCH + 1) + ": " + (batchStart+1) + "-" + batchEnd + " / " + docPages.length);
+    }
+    
+    // Merge PDFs
+    setStatus("merging");
+    const finalPdf = path.join(JOB_DIR, "output.pdf");
+    if (allPdfs.length > 0) {
+      execSync("pdfunite " + allPdfs.join(" ") + " " + finalPdf, {stdio: "pipe"});
+    }
+    
+    const pdfSize = fs.existsSync(finalPdf) ? fs.statSync(finalPdf).size : 0;
+    setStatus("done:" + pdfSize + ":" + docPages.length);
+    log("DONE. PDF: " + (pdfSize/1024/1024).toFixed(1) + "MB, MD: " + (fs.statSync(mdPath).size/1024).toFixed(0) + "KB");
+    
+  } catch(e) {
+    setStatus("error:" + e.message.slice(0, 200));
+    log("ERROR: " + e.message);
+  }
+})();
+"""
+
+HTML = """<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>URL to PDF</title>
+<style>
+*{box-sizing:border-box;margin:0;padding:0}
+body{font-family:-apple-system,system-ui,sans-serif;background:#09090b;color:#f0f0f4;min-height:100vh}
+.card{background:#131317;border:1px solid #2a2a33;border-radius:16px;padding:32px;max-width:560px;width:100%;margin:20px auto}
+h1{font-size:1.3rem;font-weight:700;margin-bottom:4px;letter-spacing:-0.3px}
+.sub{color:#88889a;font-size:0.85rem;margin-bottom:20px}
+input{width:100%;padding:12px 16px;border-radius:10px;border:1px solid #2a2a33;background:#09090b;color:#f0f0f4;font-size:0.9rem;font-family:inherit;margin-bottom:12px;transition:.2s}
+input:focus{outline:none;border-color:#8b5cf6;box-shadow:0 0 0 3px rgba(139,92,246,0.1)}
+button{width:100%;padding:12px;border-radius:10px;border:none;background:#8b5cf6;color:#fff;font-size:0.95rem;font-weight:600;cursor:pointer;font-family:inherit;transition:.2s}
+button:hover{background:#7c3aed}
+button:disabled{opacity:0.4;cursor:not-allowed}
+.tabs{display:flex;gap:8px;margin-bottom:20px}
+.tab{padding:8px 16px;border-radius:8px;cursor:pointer;background:#1a1a22;border:1px solid #2a2a33;color:#88889a;font-size:0.85rem;font-family:inherit;transition:.2s}
+.tab.active{background:#8b5cf6;border-color:#8b5cf6;color:#fff}
+.tab-content{display:none}
+.tab-content.active{display:block}
+.result{margin-top:16px;padding:12px 16px;border-radius:10px;font-size:0.85rem;text-align:center}
+.success{background:rgba(16,185,129,0.1);color:#34d399}
+.error{background:rgba(239,68,68,0.1);color:#f87171}
+.info{color:#88889a;font-size:0.75rem;margin-top:12px;text-align:center}
+.loader{display:none;text-align:center;padding:16px;color:#88889a}
+.loader.active{display:block}
+.spinner{width:24px;height:24px;border:2px solid #2a2a33;border-top-color:#8b5cf6;border-radius:50%;animation:spin .6s linear infinite;margin:0 auto 8px}
+@keyframes spin{to{transform:rotate(360deg)}}
+.progress{height:6px;background:#1a1a22;border-radius:3px;margin-top:12px;overflow:hidden}
+.progress-bar{height:100%;background:#8b5cf6;border-radius:3px;transition:width .3s;width:0}
+a.dl-btn{display:inline-block;padding:10px 16px;border-radius:10px;background:#8b5cf6;color:#fff;text-decoration:none;font-weight:600;margin:4px;font-size:0.85rem}
+a.dl-btn.green{background:#059669}
+</style></head>
+<body>
+<div class="card" style="margin-top:60px">
+<h1>PDF Tools</h1>
+<p class="sub">Convert URLs & documentation sites</p>
+
+<div class="tabs">
+<button class="tab active" onclick="switchTab('single')">Single URL</button>
+<button class="tab" onclick="switchTab('crawl')">Doc Site Crawler</button>
+</div>
+
+<!-- Single URL -->
+<div class="tab-content active" id="tab-single">
+<form id="form">
+<input id="url" type="url" placeholder="https://..." required autofocus>
+<button type="submit" id="btn">Convert to PDF</button>
+</form>
+<div class="loader" id="loader"><div class="spinner"></div>Converting...</div>
+<div id="result"></div>
+<p class="info">Powered by Chrome (Puppeteer)</p>
+</div>
+
+<!-- Doc Crawler -->
+<div class="tab-content" id="tab-crawl">
+<p class="sub" style="margin-bottom:12px">Crawl an entire documentation site into a single PDF + Markdown file</p>
+<form id="crawl-form">
+<input id="crawl-url" type="url" placeholder="Paste any doc page URL..." required>
+<button type="submit" id="crawl-btn">Start Crawling</button>
+</form>
+<div class="loader" id="crawl-loader"><div class="spinner"></div><span id="crawl-status"></span></div>
+<div class="progress"><div class="progress-bar" id="crawl-progress" style="width:0"></div></div>
+<div id="crawl-result"></div>
+<p class="info">Each doc page becomes one PDF page. Large sites may take a few minutes.</p>
+</div>
+</div>
+
+<script>
+function switchTab(name){
+  document.querySelectorAll(".tab").forEach(t=>t.classList.remove("active"));
+  document.querySelector(".tab[onclick*="+name+"]")?.classList.add("active");
+  document.getElementById("tab-single").classList.toggle("active",name==="single");
+  document.getElementById("tab-crawl").classList.toggle("active",name==="crawl");
+}
+
+// Single URL
+document.getElementById("form").addEventListener("submit",async e=>{
+e.preventDefault();
+var btn=document.getElementById("btn"),loader=document.getElementById("loader"),res=document.getElementById("result"),url=document.getElementById("url").value.trim();
+if(!url)return;
+btn.disabled=true;loader.classList.add("active");res.innerHTML="";
+try{
+var r=await fetch("/url2pdf/convert",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
+if(!r.ok)throw new Error(await r.text());
+var blob=await r.blob();
+var a=document.createElement("a");a.href=URL.createObjectURL(blob);a.download="output.pdf";a.click();
+res.innerHTML='<div class="success">PDF downloaded!</div>';
+}catch(err){res.innerHTML='<div class="error">'+err.message+"</div>"}
+btn.disabled=false;loader.classList.remove("active");
+});
+
+// Doc Crawler
+document.getElementById("crawl-form").addEventListener("submit",async e=>{
+e.preventDefault();
+var btn=document.getElementById("crawl-btn"),loader=document.getElementById("crawl-loader"),status=document.getElementById("crawl-status"),progress=document.getElementById("crawl-progress"),res=document.getElementById("crawl-result"),url=document.getElementById("crawl-url").value.trim();
+if(!url)return;
+btn.disabled=true;loader.classList.add("active");res.innerHTML="";progress.style.width="0";
+status.textContent="Starting crawl...";
+try{
+var r=await fetch("/url2pdf/crawl",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
+if(!r.ok)throw new Error(await r.text());
+var data=await r.json();
+var job=data.job;
+
+// Poll for status
+var poll=setInterval(async()=>{
+try{
+var sr=await fetch("/url2pdf/crawl/status?job="+job);
+var s=await sr.json();
+if(s.status.startsWith("processing:")){
+var parts=s.status.split(":");
+var cur=parseInt(parts[1]),total=parseInt(parts[2]);
+var pct=Math.round((cur/total)*100);
+status.textContent="Processing page "+(cur+1)+" of "+total+" ("+pct+"%)";
+progress.style.width=pct+"%";
+}else if(s.status.startsWith("done:")){
+clearInterval(poll);
+var parts=s.status.split(":");
+status.textContent="Done! "+parts[2]+" pages";
+progress.style.width="100%";
+btn.disabled=false;loader.classList.remove("active");
+res.innerHTML='<a class="dl-btn" href="/url2pdf/crawl/download?job='+job+'&format=pdf">Download PDF</a> <a class="dl-btn green" href="/url2pdf/crawl/download?job='+job+'&format=md">Download Markdown</a>';
+}else if(s.status.startsWith("error:")){
+clearInterval(poll);
+status.textContent=s.status;
+btn.disabled=false;loader.classList.remove("active");
+res.innerHTML='<div class="error">'+s.status+"</div>";
+}else{
+status.textContent=s.status;
+}
+}catch(err){}
+},2000);
+}catch(err){
+btn.disabled=false;loader.classList.remove("active");
+res.innerHTML='<div class="error">'+err.message+"</div>";
+}
+});
+</script>
+</body></html>"""
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass  # silence logs
+    
+    def _json(self, code, data):
+        self.send_response(code)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(data).encode())
+    
+    def do_GET(self):
+        if self.path.startswith("/crawl/status"):
+            qs = urllib.parse.urlparse(self.path).query
+            params = urllib.parse.parse_qs(qs)
+            job = params.get("job", [None])[0]
+            if not job:
+                self._json(400, {"error": "Missing job"})
+                return
+            sp = os.path.join(JOBS_DIR, job, "status.json")
+            if os.path.exists(sp):
+                with open(sp) as f:
+                    self._json(200, json.load(f))
+            else:
+                self._json(200, {"status": "pending"})
+            return
+        
+        if self.path.startswith("/crawl/download"):
+            qs = urllib.parse.urlparse(self.path).query
+            params = urllib.parse.parse_qs(qs)
+            job = params.get("job", [None])[0]
+            fmt = params.get("format", ["pdf"])[0]
+            if not job:
+                self.send_error(400)
+                return
+            
+            jdir = os.path.join(JOBS_DIR, job)
+            if fmt == "md":
+                fp = os.path.join(jdir, "output.md")
+                ct = "text/markdown"
+            else:
+                fp = os.path.join(jdir, "output.pdf")
+                ct = "application/pdf"
+            
+            if not os.path.exists(fp):
+                self.send_error(404, "Not found")
+                return
+            
+            with open(fp, "rb") as f:
+                data = f.read()
+            self.send_response(200)
+            self.send_header("Content-Type", ct)
+            self.send_header("Content-Disposition", f'attachment; filename="{job}.{fmt}"')
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+        
+        # Serve HTML
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html")
+        self.end_headers()
+        self.wfile.write(HTML.encode())
+    
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length", 0))
+        body = self.rfile.read(length)
+        try:
+            data = json.loads(body)
+        except:
+            self.send_error(400, "Bad JSON")
+            return
+        
+        if self.path == "/convert":
+            url = data.get("url", "").strip()
+            if not url:
+                self.send_error(400, "No URL")
+                return
+            parsed = urllib.parse.urlparse(url)
+            if parsed.scheme not in ("http", "https"):
+                self.send_error(400, "Invalid URL scheme")
+                return
+            
+            js_path = os.path.join(CWD, "convert.js")
+            if not os.path.exists(js_path):
+                with open(js_path, "w") as f:
+                    f.write(PUPPETEER_JS)
+            
+            fd, tmp = tempfile.mkstemp(suffix=".pdf")
+            os.close(fd)
+            try:
+                r = subprocess.run(
+                    ["timeout", "40", "node", js_path, url, tmp],
+                    capture_output=True, text=True, timeout=45, cwd=CWD
+                )
+                if r.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) < 100:
+                    raise Exception("Conversion failed: " + r.stderr[:200])
+                
+                with open(tmp, "rb") as f:
+                    pdf_data = f.read()
+                self.send_response(200)
+                self.send_header("Content-Type", "application/pdf")
+                self.send_header("Content-Disposition", 'attachment; filename="output.pdf"')
+                self.send_header("Content-Length", str(len(pdf_data)))
+                self.end_headers()
+                self.wfile.write(pdf_data)
+            except Exception as e:
+                self.send_error(500, str(e))
+            finally:
+                if os.path.exists(tmp):
+                    os.unlink(tmp)
+        
+        elif self.path == "/crawl":
+            url = data.get("url", "").strip()
+            if not url:
+                self._json(400, {"error": "No URL"})
+                return
+            
+            import uuid
+            job = uuid.uuid4().hex[:12]
+            jdir = os.path.join(JOBS_DIR, job)
+            os.makedirs(jdir, exist_ok=True)
+            
+            # Write crawler script to npm dir so require() works
+            crawl_js = os.path.join(CWD, f"crawler_{job}.js")
+            with open(crawl_js, "w") as f:
+                f.write(CRAWLER_JS)
+            
+            # Write initial status
+            with open(os.path.join(jdir, "status.json"), "w") as f:
+                json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
+            
+            # Start background crawler
+            def run():
+                subprocess.run(
+                    ["timeout", "600", "node", crawl_js, url, jdir],
+                    capture_output=True, text=True, cwd=CWD,
+                    env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
+                )
+                # Clean up script
+                try: os.unlink(crawl_js)
+                except: pass
+            
+            t = threading.Thread(target=run, daemon=True)
+            t.start()
+            
+            self._json(200, {"job": job, "status": "started"})
+        
+        else:
+            self.send_error(404)
+
+HTTPServer(("127.0.0.1", 8099), Handler).serve_forever()