IT Dog
0d3e40f797
- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
456 lines
19 KiB
Python
456 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
import subprocess, tempfile, os, json, urllib.parse, time, threading, shutil
|
|
|
|
JOBS_DIR = "/tmp/url2pdf_jobs"
|
|
os.makedirs(JOBS_DIR, exist_ok=True)
|
|
CWD = "/opt/url2pdf"
|
|
|
|
PUPPETEER_JS = """
|
|
const puppeteer = require("puppeteer");
|
|
(async () => {
|
|
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]});
|
|
const page = await browser.newPage();
|
|
await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000});
|
|
await page.pdf({path: process.argv[3], format: "A4", printBackground: true});
|
|
await browser.close();
|
|
})();
|
|
"""
|
|
|
|
CRAWLER_JS = """
|
|
// Documentation crawler - cwd must have puppeteer installed
|
|
const puppeteer = require("puppeteer");
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const { execSync } = require("child_process");
|
|
|
|
const BASE = process.argv[2];
|
|
const JOB_DIR = process.argv[3];
|
|
const baseUrl = new URL(BASE);
|
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf("/") + 1);
|
|
|
|
function log(msg) {
|
|
fs.appendFileSync(path.join(JOB_DIR, "log.txt"), msg + "\\n");
|
|
process.stdout.write(msg + "\\n");
|
|
}
|
|
|
|
(async () => {
|
|
const statusPath = path.join(JOB_DIR, "status.json");
|
|
function setStatus(s) {
|
|
fs.writeFileSync(statusPath, JSON.stringify({status: s, time: new Date().toISOString()}));
|
|
}
|
|
|
|
try {
|
|
setStatus("extracting_urls");
|
|
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
|
const mainPage = await browser.newPage();
|
|
|
|
await mainPage.goto(BASE, {waitUntil: "networkidle2", timeout: 30000});
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
|
|
const allLinks = await mainPage.evaluate(() => {
|
|
const links = [];
|
|
document.querySelectorAll("a[href]").forEach(a => {
|
|
const text = a.textContent.trim();
|
|
if (text.length > 2) links.push({href: a.href, text: text.slice(0, 80)});
|
|
});
|
|
return links;
|
|
});
|
|
await mainPage.close();
|
|
await browser.close();
|
|
|
|
// Filter unique doc pages (en-us only, dedupe by filename, skip language switchers)
|
|
const seenFiles = new Set();
|
|
const docPages = [];
|
|
for (const link of allLinks) {
|
|
try {
|
|
const u = new URL(link.href);
|
|
if (u.hostname !== baseUrl.hostname) continue;
|
|
if (!u.pathname.startsWith(articleDir)) continue;
|
|
if (!u.pathname.startsWith("/en-us/")) continue;
|
|
if (u.hash) continue;
|
|
const fn = u.pathname.split("/").pop();
|
|
if (!fn || fn.length < 20) continue;
|
|
const lower = link.text.toLowerCase();
|
|
if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
|
|
if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
|
|
if (!seenFiles.has(fn)) {
|
|
seenFiles.add(fn);
|
|
docPages.push({href: link.href, text: link.text});
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
|
|
log("Found " + docPages.length + " pages");
|
|
setStatus("crawling:" + docPages.length);
|
|
|
|
// Process in batches of 20 to avoid OOM
|
|
const BATCH = 20;
|
|
const mdPath = path.join(JOB_DIR, "output.md");
|
|
const pdfDir = path.join(JOB_DIR, "pdfs");
|
|
fs.mkdirSync(pdfDir, {recursive: true});
|
|
|
|
// Write MD header
|
|
fs.writeFileSync(mdPath, "# " + baseUrl.hostname.split(".")[0] + " Documentation\\n\\n> " + docPages.length + " pages from " + BASE + "\\n\\n");
|
|
|
|
const allPdfs = [];
|
|
|
|
for (let batchStart = 0; batchStart < docPages.length; batchStart += BATCH) {
|
|
const batchEnd = Math.min(batchStart + BATCH, docPages.length);
|
|
setStatus("processing:" + batchStart + "/" + docPages.length);
|
|
|
|
const b = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
|
|
|
for (let i = batchStart; i < batchEnd; i++) {
|
|
const {href, text} = docPages[i];
|
|
try {
|
|
const p = await b.newPage();
|
|
await p.goto(href, {waitUntil: "networkidle2", timeout: 20000});
|
|
await new Promise(r => setTimeout(r, 300));
|
|
|
|
// Extract text for MD
|
|
const content = await p.evaluate(() => {
|
|
const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
|
|
let main = document.body;
|
|
for (const sel of mainSel) { const el = document.querySelector(sel); if (el) { main = el; break; } }
|
|
const h1 = main.querySelector("h1, h2");
|
|
const title = h1 ? h1.textContent.trim() : document.title.replace(/ \\| .*$/, "").trim();
|
|
const blocks = [];
|
|
main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
|
|
const txt = el.textContent.trim();
|
|
if (!txt || txt.length < 3) return;
|
|
const tag = el.tagName;
|
|
if (tag.match(/^H[1-4]$/)) blocks.push("\\n" + "#".repeat(parseInt(tag[1])+1) + " " + txt + "\\n");
|
|
else if (tag === "LI") blocks.push("- " + txt);
|
|
else if (tag === "PRE") blocks.push("\\n```\\n" + txt + "\\n```\\n");
|
|
else blocks.push(txt);
|
|
});
|
|
return {title, body: blocks.join("\\n\\n")};
|
|
});
|
|
|
|
fs.appendFileSync(mdPath, "## " + (i+1) + ". " + content.title + "\\n\\n" + (content.body || "_") + "\\n\\n");
|
|
|
|
// PDF for this page
|
|
const pdfFile = path.join(pdfDir, String(i).padStart(4, "0") + ".pdf");
|
|
await p.pdf({path: pdfFile, format: "A4", printBackground: true});
|
|
allPdfs.push(pdfFile);
|
|
await p.close();
|
|
} catch(e) {
|
|
fs.appendFileSync(mdPath, "## " + (i+1) + ". " + text + "\\n\\n_Error_\\n\\n");
|
|
}
|
|
}
|
|
|
|
await b.close();
|
|
log("Batch " + (batchStart/BATCH + 1) + ": " + (batchStart+1) + "-" + batchEnd + " / " + docPages.length);
|
|
}
|
|
|
|
// Merge PDFs
|
|
setStatus("merging");
|
|
const finalPdf = path.join(JOB_DIR, "output.pdf");
|
|
if (allPdfs.length > 0) {
|
|
execSync("pdfunite " + allPdfs.join(" ") + " " + finalPdf, {stdio: "pipe"});
|
|
}
|
|
|
|
const pdfSize = fs.existsSync(finalPdf) ? fs.statSync(finalPdf).size : 0;
|
|
setStatus("done:" + pdfSize + ":" + docPages.length);
|
|
log("DONE. PDF: " + (pdfSize/1024/1024).toFixed(1) + "MB, MD: " + (fs.statSync(mdPath).size/1024).toFixed(0) + "KB");
|
|
|
|
} catch(e) {
|
|
setStatus("error:" + e.message.slice(0, 200));
|
|
log("ERROR: " + e.message);
|
|
}
|
|
})();
|
|
"""
|
|
|
|
HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>URL to PDF</title>
|
|
<style>
|
|
*{box-sizing:border-box;margin:0;padding:0}
|
|
body{font-family:-apple-system,system-ui,sans-serif;background:#09090b;color:#f0f0f4;min-height:100vh}
|
|
.card{background:#131317;border:1px solid #2a2a33;border-radius:16px;padding:32px;max-width:560px;width:100%;margin:20px auto}
|
|
h1{font-size:1.3rem;font-weight:700;margin-bottom:4px;letter-spacing:-0.3px}
|
|
.sub{color:#88889a;font-size:0.85rem;margin-bottom:20px}
|
|
input{width:100%;padding:12px 16px;border-radius:10px;border:1px solid #2a2a33;background:#09090b;color:#f0f0f4;font-size:0.9rem;font-family:inherit;margin-bottom:12px;transition:.2s}
|
|
input:focus{outline:none;border-color:#8b5cf6;box-shadow:0 0 0 3px rgba(139,92,246,0.1)}
|
|
button{width:100%;padding:12px;border-radius:10px;border:none;background:#8b5cf6;color:#fff;font-size:0.95rem;font-weight:600;cursor:pointer;font-family:inherit;transition:.2s}
|
|
button:hover{background:#7c3aed}
|
|
button:disabled{opacity:0.4;cursor:not-allowed}
|
|
.tabs{display:flex;gap:8px;margin-bottom:20px}
|
|
.tab{padding:8px 16px;border-radius:8px;cursor:pointer;background:#1a1a22;border:1px solid #2a2a33;color:#88889a;font-size:0.85rem;font-family:inherit;transition:.2s}
|
|
.tab.active{background:#8b5cf6;border-color:#8b5cf6;color:#fff}
|
|
.tab-content{display:none}
|
|
.tab-content.active{display:block}
|
|
.result{margin-top:16px;padding:12px 16px;border-radius:10px;font-size:0.85rem;text-align:center}
|
|
.success{background:rgba(16,185,129,0.1);color:#34d399}
|
|
.error{background:rgba(239,68,68,0.1);color:#f87171}
|
|
.info{color:#88889a;font-size:0.75rem;margin-top:12px;text-align:center}
|
|
.loader{display:none;text-align:center;padding:16px;color:#88889a}
|
|
.loader.active{display:block}
|
|
.spinner{width:24px;height:24px;border:2px solid #2a2a33;border-top-color:#8b5cf6;border-radius:50%;animation:spin .6s linear infinite;margin:0 auto 8px}
|
|
@keyframes spin{to{transform:rotate(360deg)}}
|
|
.progress{height:6px;background:#1a1a22;border-radius:3px;margin-top:12px;overflow:hidden}
|
|
.progress-bar{height:100%;background:#8b5cf6;border-radius:3px;transition:width .3s;width:0}
|
|
a.dl-btn{display:inline-block;padding:10px 16px;border-radius:10px;background:#8b5cf6;color:#fff;text-decoration:none;font-weight:600;margin:4px;font-size:0.85rem}
|
|
a.dl-btn.green{background:#059669}
|
|
</style></head>
|
|
<body>
|
|
<div class="card" style="margin-top:60px">
|
|
<h1>PDF Tools</h1>
|
|
<p class="sub">Convert URLs & documentation sites</p>
|
|
|
|
<div class="tabs">
|
|
<button class="tab active" onclick="switchTab('single')">Single URL</button>
|
|
<button class="tab" onclick="switchTab('crawl')">Doc Site Crawler</button>
|
|
</div>
|
|
|
|
<!-- Single URL -->
|
|
<div class="tab-content active" id="tab-single">
|
|
<form id="form">
|
|
<input id="url" type="url" placeholder="https://..." required autofocus>
|
|
<button type="submit" id="btn">Convert to PDF</button>
|
|
</form>
|
|
<div class="loader" id="loader"><div class="spinner"></div>Converting...</div>
|
|
<div id="result"></div>
|
|
<p class="info">Powered by Chrome (Puppeteer)</p>
|
|
</div>
|
|
|
|
<!-- Doc Crawler -->
|
|
<div class="tab-content" id="tab-crawl">
|
|
<p class="sub" style="margin-bottom:12px">Crawl an entire documentation site into a single PDF + Markdown file</p>
|
|
<form id="crawl-form">
|
|
<input id="crawl-url" type="url" placeholder="Paste any doc page URL..." required>
|
|
<button type="submit" id="crawl-btn">Start Crawling</button>
|
|
</form>
|
|
<div class="loader" id="crawl-loader"><div class="spinner"></div><span id="crawl-status"></span></div>
|
|
<div class="progress"><div class="progress-bar" id="crawl-progress" style="width:0"></div></div>
|
|
<div id="crawl-result"></div>
|
|
<p class="info">Each doc page becomes one PDF page. Large sites may take a few minutes.</p>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
function switchTab(name){
|
|
document.querySelectorAll(".tab").forEach(t=>t.classList.remove("active"));
|
|
document.querySelector(".tab[onclick*="+name+"]")?.classList.add("active");
|
|
document.getElementById("tab-single").classList.toggle("active",name==="single");
|
|
document.getElementById("tab-crawl").classList.toggle("active",name==="crawl");
|
|
}
|
|
|
|
// Single URL
|
|
document.getElementById("form").addEventListener("submit",async e=>{
|
|
e.preventDefault();
|
|
var btn=document.getElementById("btn"),loader=document.getElementById("loader"),res=document.getElementById("result"),url=document.getElementById("url").value.trim();
|
|
if(!url)return;
|
|
btn.disabled=true;loader.classList.add("active");res.innerHTML="";
|
|
try{
|
|
var r=await fetch("/url2pdf/convert",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
|
|
if(!r.ok)throw new Error(await r.text());
|
|
var blob=await r.blob();
|
|
var a=document.createElement("a");a.href=URL.createObjectURL(blob);a.download="output.pdf";a.click();
|
|
res.innerHTML='<div class="success">PDF downloaded!</div>';
|
|
}catch(err){res.innerHTML='<div class="error">'+err.message+"</div>"}
|
|
btn.disabled=false;loader.classList.remove("active");
|
|
});
|
|
|
|
// Doc Crawler
|
|
document.getElementById("crawl-form").addEventListener("submit",async e=>{
|
|
e.preventDefault();
|
|
var btn=document.getElementById("crawl-btn"),loader=document.getElementById("crawl-loader"),status=document.getElementById("crawl-status"),progress=document.getElementById("crawl-progress"),res=document.getElementById("crawl-result"),url=document.getElementById("crawl-url").value.trim();
|
|
if(!url)return;
|
|
btn.disabled=true;loader.classList.add("active");res.innerHTML="";progress.style.width="0";
|
|
status.textContent="Starting crawl...";
|
|
try{
|
|
var r=await fetch("/url2pdf/crawl",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
|
|
if(!r.ok)throw new Error(await r.text());
|
|
var data=await r.json();
|
|
var job=data.job;
|
|
|
|
// Poll for status
|
|
var poll=setInterval(async()=>{
|
|
try{
|
|
var sr=await fetch("/url2pdf/crawl/status?job="+job);
|
|
var s=await sr.json();
|
|
if(s.status.startsWith("processing:")){
|
|
var parts=s.status.split(":");
|
|
var cur=parseInt(parts[1]),total=parseInt(parts[2]);
|
|
var pct=Math.round((cur/total)*100);
|
|
status.textContent="Processing page "+(cur+1)+" of "+total+" ("+pct+"%)";
|
|
progress.style.width=pct+"%";
|
|
}else if(s.status.startsWith("done:")){
|
|
clearInterval(poll);
|
|
var parts=s.status.split(":");
|
|
status.textContent="Done! "+parts[2]+" pages";
|
|
progress.style.width="100%";
|
|
btn.disabled=false;loader.classList.remove("active");
|
|
res.innerHTML='<a class="dl-btn" href="/url2pdf/crawl/download?job='+job+'&format=pdf">Download PDF</a> <a class="dl-btn green" href="/url2pdf/crawl/download?job='+job+'&format=md">Download Markdown</a>';
|
|
}else if(s.status.startsWith("error:")){
|
|
clearInterval(poll);
|
|
status.textContent=s.status;
|
|
btn.disabled=false;loader.classList.remove("active");
|
|
res.innerHTML='<div class="error">'+s.status+"</div>";
|
|
}else{
|
|
status.textContent=s.status;
|
|
}
|
|
}catch(err){}
|
|
},2000);
|
|
}catch(err){
|
|
btn.disabled=false;loader.classList.remove("active");
|
|
res.innerHTML='<div class="error">'+err.message+"</div>";
|
|
}
|
|
});
|
|
</script>
|
|
</body></html>"""
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, format, *args):
|
|
pass # silence logs
|
|
|
|
def _json(self, code, data):
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.end_headers()
|
|
self.wfile.write(json.dumps(data).encode())
|
|
|
|
def do_GET(self):
|
|
if self.path.startswith("/crawl/status"):
|
|
qs = urllib.parse.urlparse(self.path).query
|
|
params = urllib.parse.parse_qs(qs)
|
|
job = params.get("job", [None])[0]
|
|
if not job:
|
|
self._json(400, {"error": "Missing job"})
|
|
return
|
|
sp = os.path.join(JOBS_DIR, job, "status.json")
|
|
if os.path.exists(sp):
|
|
with open(sp) as f:
|
|
self._json(200, json.load(f))
|
|
else:
|
|
self._json(200, {"status": "pending"})
|
|
return
|
|
|
|
if self.path.startswith("/crawl/download"):
|
|
qs = urllib.parse.urlparse(self.path).query
|
|
params = urllib.parse.parse_qs(qs)
|
|
job = params.get("job", [None])[0]
|
|
fmt = params.get("format", ["pdf"])[0]
|
|
if not job:
|
|
self.send_error(400)
|
|
return
|
|
|
|
jdir = os.path.join(JOBS_DIR, job)
|
|
if fmt == "md":
|
|
fp = os.path.join(jdir, "output.md")
|
|
ct = "text/markdown"
|
|
else:
|
|
fp = os.path.join(jdir, "output.pdf")
|
|
ct = "application/pdf"
|
|
|
|
if not os.path.exists(fp):
|
|
self.send_error(404, "Not found")
|
|
return
|
|
|
|
with open(fp, "rb") as f:
|
|
data = f.read()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", ct)
|
|
self.send_header("Content-Disposition", f'attachment; filename="{job}.{fmt}"')
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
return
|
|
|
|
# Serve HTML
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html")
|
|
self.end_headers()
|
|
self.wfile.write(HTML.encode())
|
|
|
|
def do_POST(self):
|
|
length = int(self.headers.get("Content-Length", 0))
|
|
body = self.rfile.read(length)
|
|
try:
|
|
data = json.loads(body)
|
|
except:
|
|
self.send_error(400, "Bad JSON")
|
|
return
|
|
|
|
if self.path == "/convert":
|
|
url = data.get("url", "").strip()
|
|
if not url:
|
|
self.send_error(400, "No URL")
|
|
return
|
|
parsed = urllib.parse.urlparse(url)
|
|
if parsed.scheme not in ("http", "https"):
|
|
self.send_error(400, "Invalid URL scheme")
|
|
return
|
|
|
|
js_path = os.path.join(CWD, "convert.js")
|
|
if not os.path.exists(js_path):
|
|
with open(js_path, "w") as f:
|
|
f.write(PUPPETEER_JS)
|
|
|
|
fd, tmp = tempfile.mkstemp(suffix=".pdf")
|
|
os.close(fd)
|
|
try:
|
|
r = subprocess.run(
|
|
["timeout", "40", "node", js_path, url, tmp],
|
|
capture_output=True, text=True, timeout=45, cwd=CWD
|
|
)
|
|
if r.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) < 100:
|
|
raise Exception("Conversion failed: " + r.stderr[:200])
|
|
|
|
with open(tmp, "rb") as f:
|
|
pdf_data = f.read()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/pdf")
|
|
self.send_header("Content-Disposition", 'attachment; filename="output.pdf"')
|
|
self.send_header("Content-Length", str(len(pdf_data)))
|
|
self.end_headers()
|
|
self.wfile.write(pdf_data)
|
|
except Exception as e:
|
|
self.send_error(500, str(e))
|
|
finally:
|
|
if os.path.exists(tmp):
|
|
os.unlink(tmp)
|
|
|
|
elif self.path == "/crawl":
|
|
url = data.get("url", "").strip()
|
|
if not url:
|
|
self._json(400, {"error": "No URL"})
|
|
return
|
|
|
|
import uuid
|
|
job = uuid.uuid4().hex[:12]
|
|
jdir = os.path.join(JOBS_DIR, job)
|
|
os.makedirs(jdir, exist_ok=True)
|
|
|
|
# Write crawler script to npm dir so require() works
|
|
crawl_js = os.path.join(CWD, f"crawler_{job}.js")
|
|
with open(crawl_js, "w") as f:
|
|
f.write(CRAWLER_JS)
|
|
|
|
# Write initial status
|
|
with open(os.path.join(jdir, "status.json"), "w") as f:
|
|
json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
|
|
|
|
# Start background crawler
|
|
def run():
|
|
subprocess.run(
|
|
["timeout", "600", "node", crawl_js, url, jdir],
|
|
capture_output=True, text=True, cwd=CWD,
|
|
env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
|
|
)
|
|
# Clean up script
|
|
try: os.unlink(crawl_js)
|
|
except: pass
|
|
|
|
t = threading.Thread(target=run, daemon=True)
|
|
t.start()
|
|
|
|
self._json(200, {"job": job, "status": "started"})
|
|
|
|
else:
|
|
self.send_error(404)
|
|
|
|
HTTPServer(("127.0.0.1", 8099), Handler).serve_forever()
|