#!/usr/bin/env python3 from http.server import HTTPServer, BaseHTTPRequestHandler import subprocess, tempfile, os, json, urllib.parse, time, threading, shutil JOBS_DIR = "/tmp/url2pdf_jobs" os.makedirs(JOBS_DIR, exist_ok=True) CWD = "/opt/url2pdf" PUPPETEER_JS = """ const puppeteer = require("puppeteer"); (async () => { const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]}); const page = await browser.newPage(); await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000}); await page.pdf({path: process.argv[3], format: "A4", printBackground: true}); await browser.close(); })(); """ CRAWLER_JS = """ // Documentation crawler - cwd must have puppeteer installed const puppeteer = require("puppeteer"); const fs = require("fs"); const path = require("path"); const { execSync } = require("child_process"); const BASE = process.argv[2]; const JOB_DIR = process.argv[3]; const baseUrl = new URL(BASE); const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf("/") + 1); function log(msg) { fs.appendFileSync(path.join(JOB_DIR, "log.txt"), msg + "\\n"); process.stdout.write(msg + "\\n"); } (async () => { const statusPath = path.join(JOB_DIR, "status.json"); function setStatus(s) { fs.writeFileSync(statusPath, JSON.stringify({status: s, time: new Date().toISOString()})); } try { setStatus("extracting_urls"); const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); const mainPage = await browser.newPage(); await mainPage.goto(BASE, {waitUntil: "networkidle2", timeout: 30000}); await new Promise(r => setTimeout(r, 3000)); const allLinks = await mainPage.evaluate(() => { const links = []; document.querySelectorAll("a[href]").forEach(a => { const text = a.textContent.trim(); if (text.length > 2) links.push({href: a.href, text: text.slice(0, 80)}); }); return links; }); await mainPage.close(); await browser.close(); // Filter unique doc pages (en-us only, dedupe by filename, skip language switchers) const seenFiles = new Set(); const docPages = []; for (const link of allLinks) { try { const u = new URL(link.href); if (u.hostname !== baseUrl.hostname) continue; if (!u.pathname.startsWith(articleDir)) continue; if (!u.pathname.startsWith("/en-us/")) continue; if (u.hash) continue; const fn = u.pathname.split("/").pop(); if (!fn || fn.length < 20) continue; const lower = link.text.toLowerCase(); if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue; if (/online help center|security for endpoints|apex one$/i.test(lower)) continue; if (!seenFiles.has(fn)) { seenFiles.add(fn); docPages.push({href: link.href, text: link.text}); } } catch(e) {} } log("Found " + docPages.length + " pages"); setStatus("crawling:" + docPages.length); // Process in batches of 20 to avoid OOM const BATCH = 20; const mdPath = path.join(JOB_DIR, "output.md"); const pdfDir = path.join(JOB_DIR, "pdfs"); fs.mkdirSync(pdfDir, {recursive: true}); // Write MD header fs.writeFileSync(mdPath, "# " + baseUrl.hostname.split(".")[0] + " Documentation\\n\\n> " + docPages.length + " pages from " + BASE + "\\n\\n"); const allPdfs = []; for (let batchStart = 0; batchStart < docPages.length; batchStart += BATCH) { const batchEnd = Math.min(batchStart + BATCH, docPages.length); setStatus("processing:" + batchStart + "/" + docPages.length); const b = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); for (let i = batchStart; i < batchEnd; i++) { const {href, text} = docPages[i]; try { const p = await b.newPage(); await p.goto(href, {waitUntil: "networkidle2", timeout: 20000}); await new Promise(r => setTimeout(r, 300)); // Extract text for MD const content = await p.evaluate(() => { const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"]; let main = document.body; for (const sel of mainSel) { const el = document.querySelector(sel); if (el) { main = el; break; } } const h1 = main.querySelector("h1, h2"); const title = h1 ? h1.textContent.trim() : document.title.replace(/ \\| .*$/, "").trim(); const blocks = []; main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => { const txt = el.textContent.trim(); if (!txt || txt.length < 3) return; const tag = el.tagName; if (tag.match(/^H[1-4]$/)) blocks.push("\\n" + "#".repeat(parseInt(tag[1])+1) + " " + txt + "\\n"); else if (tag === "LI") blocks.push("- " + txt); else if (tag === "PRE") blocks.push("\\n```\\n" + txt + "\\n```\\n"); else blocks.push(txt); }); return {title, body: blocks.join("\\n\\n")}; }); fs.appendFileSync(mdPath, "## " + (i+1) + ". " + content.title + "\\n\\n" + (content.body || "_") + "\\n\\n"); // PDF for this page const pdfFile = path.join(pdfDir, String(i).padStart(4, "0") + ".pdf"); await p.pdf({path: pdfFile, format: "A4", printBackground: true}); allPdfs.push(pdfFile); await p.close(); } catch(e) { fs.appendFileSync(mdPath, "## " + (i+1) + ". " + text + "\\n\\n_Error_\\n\\n"); } } await b.close(); log("Batch " + (batchStart/BATCH + 1) + ": " + (batchStart+1) + "-" + batchEnd + " / " + docPages.length); } // Merge PDFs setStatus("merging"); const finalPdf = path.join(JOB_DIR, "output.pdf"); if (allPdfs.length > 0) { execSync("pdfunite " + allPdfs.join(" ") + " " + finalPdf, {stdio: "pipe"}); } const pdfSize = fs.existsSync(finalPdf) ? fs.statSync(finalPdf).size : 0; setStatus("done:" + pdfSize + ":" + docPages.length); log("DONE. PDF: " + (pdfSize/1024/1024).toFixed(1) + "MB, MD: " + (fs.statSync(mdPath).size/1024).toFixed(0) + "KB"); } catch(e) { setStatus("error:" + e.message.slice(0, 200)); log("ERROR: " + e.message); } })(); """ HTML = """ URL to PDF

PDF Tools

Convert URLs & documentation sites

Converting...

Powered by Chrome (Puppeteer)

Crawl an entire documentation site into a single PDF + Markdown file

Each doc page becomes one PDF page. Large sites may take a few minutes.

""" class Handler(BaseHTTPRequestHandler): def log_message(self, format, *args): pass # silence logs def _json(self, code, data): self.send_response(code) self.send_header("Content-Type", "application/json") self.end_headers() self.wfile.write(json.dumps(data).encode()) def do_GET(self): if self.path.startswith("/crawl/status"): qs = urllib.parse.urlparse(self.path).query params = urllib.parse.parse_qs(qs) job = params.get("job", [None])[0] if not job: self._json(400, {"error": "Missing job"}) return sp = os.path.join(JOBS_DIR, job, "status.json") if os.path.exists(sp): with open(sp) as f: self._json(200, json.load(f)) else: self._json(200, {"status": "pending"}) return if self.path.startswith("/crawl/download"): qs = urllib.parse.urlparse(self.path).query params = urllib.parse.parse_qs(qs) job = params.get("job", [None])[0] fmt = params.get("format", ["pdf"])[0] if not job: self.send_error(400) return jdir = os.path.join(JOBS_DIR, job) if fmt == "md": fp = os.path.join(jdir, "output.md") ct = "text/markdown" else: fp = os.path.join(jdir, "output.pdf") ct = "application/pdf" if not os.path.exists(fp): self.send_error(404, "Not found") return with open(fp, "rb") as f: data = f.read() self.send_response(200) self.send_header("Content-Type", ct) self.send_header("Content-Disposition", f'attachment; filename="{job}.{fmt}"') self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) return # Serve HTML self.send_response(200) self.send_header("Content-Type", "text/html") self.end_headers() self.wfile.write(HTML.encode()) def do_POST(self): length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) try: data = json.loads(body) except: self.send_error(400, "Bad JSON") return if self.path == "/convert": url = data.get("url", "").strip() if not url: self.send_error(400, "No URL") return parsed = urllib.parse.urlparse(url) if parsed.scheme not in ("http", "https"): self.send_error(400, "Invalid URL scheme") return js_path = os.path.join(CWD, "convert.js") if not os.path.exists(js_path): with open(js_path, "w") as f: f.write(PUPPETEER_JS) fd, tmp = tempfile.mkstemp(suffix=".pdf") os.close(fd) try: r = subprocess.run( ["timeout", "40", "node", js_path, url, tmp], capture_output=True, text=True, timeout=45, cwd=CWD ) if r.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) < 100: raise Exception("Conversion failed: " + r.stderr[:200]) with open(tmp, "rb") as f: pdf_data = f.read() self.send_response(200) self.send_header("Content-Type", "application/pdf") self.send_header("Content-Disposition", 'attachment; filename="output.pdf"') self.send_header("Content-Length", str(len(pdf_data))) self.end_headers() self.wfile.write(pdf_data) except Exception as e: self.send_error(500, str(e)) finally: if os.path.exists(tmp): os.unlink(tmp) elif self.path == "/crawl": url = data.get("url", "").strip() if not url: self._json(400, {"error": "No URL"}) return import uuid job = uuid.uuid4().hex[:12] jdir = os.path.join(JOBS_DIR, job) os.makedirs(jdir, exist_ok=True) # Write crawler script to npm dir so require() works crawl_js = os.path.join(CWD, f"crawler_{job}.js") with open(crawl_js, "w") as f: f.write(CRAWLER_JS) # Write initial status with open(os.path.join(jdir, "status.json"), "w") as f: json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f) # Start background crawler def run(): subprocess.run( ["timeout", "600", "node", crawl_js, url, jdir], capture_output=True, text=True, cwd=CWD, env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")} ) # Clean up script try: os.unlink(crawl_js) except: pass t = threading.Thread(target=run, daemon=True) t.start() self._json(200, {"job": job, "status": "started"}) else: self.send_error(404) HTTPServer(("127.0.0.1", 8099), Handler).serve_forever()