v1.0.0: URL-to-PDF + Doc Site Crawler
- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
This commit is contained in:
IT Dog
@@ -0,0 +1,3 @@
|
||||
node_modules/
|
||||
*.pdf
|
||||
/tmp/
|
||||
@@ -0,0 +1,9 @@
|
||||
|
||||
const puppeteer = require("puppeteer");
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000});
|
||||
await page.pdf({path: process.argv[3], format: "A4", printBackground: true});
|
||||
await browser.close();
|
||||
})();
|
||||
@@ -0,0 +1,142 @@
|
||||
// Crawl a Trend Micro documentation site (or similar JS doc sites)
|
||||
// Deduplicates language variants, filters out nav/UI links
|
||||
const puppeteer = require("puppeteer");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { execSync } = require("child_process");
|
||||
|
||||
const BASE = process.argv[2];
|
||||
if (!BASE) { console.error("Usage: node crawl.js <doc-page-url>"); process.exit(1); }
|
||||
|
||||
const OUT_DIR = "/tmp/doc_pages";
|
||||
const FINAL = "/tmp/merged_guide.pdf";
|
||||
const baseUrl = new URL(BASE);
|
||||
|
||||
// Extract the documentation set prefix from URL
|
||||
// e.g. /en-us/documentation/article/apex-one-service-pack-1-2025-server-online-help-
|
||||
const articleIdx = baseUrl.pathname.lastIndexOf("/");
|
||||
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
|
||||
|
||||
(async () => {
|
||||
fs.rmSync(OUT_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
|
||||
const page = await browser.newPage();
|
||||
|
||||
console.log("🌐 Loading:", BASE);
|
||||
await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
// Extract ALL links from the fully loaded page
|
||||
const allLinks = await page.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) {
|
||||
links.push({ href: a.href, text: text.slice(0, 80) });
|
||||
}
|
||||
});
|
||||
return links;
|
||||
});
|
||||
|
||||
// Filter: same host, under same article directory, not a hash link
|
||||
// Also: unique by page filename (dedupe language variants)
|
||||
const seenFiles = new Set();
|
||||
const docPages = [];
|
||||
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (u.hash) continue;
|
||||
|
||||
const pathname = u.pathname;
|
||||
// Must be under the same article directory
|
||||
if (!pathname.startsWith(articleDir)) continue;
|
||||
|
||||
// Get the page filename
|
||||
const filename = pathname.split("/").pop();
|
||||
// Must match the doc page pattern (long hyphenated names ending with article ID)
|
||||
if (!filename || filename.length < 20) continue;
|
||||
|
||||
// Skip language-switch duplicates (same filename, different language path)
|
||||
// The paths are like: /en-us/..., /fr-fr/..., /de-de/..., etc.
|
||||
// Filter to only English (en-us) pages
|
||||
if (!pathname.startsWith("/en-us/")) continue;
|
||||
|
||||
if (!seenFiles.has(filename)) {
|
||||
seenFiles.add(filename);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
console.log(`📋 Found ${docPages.length} unique English doc pages`);
|
||||
if (docPages.length === 0) {
|
||||
console.log("⚠️ No pages found with strict filter. Trying broader filter...");
|
||||
// Retry with just articleDir prefix, no language filter
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (u.hash) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
const filename = u.pathname.split("/").pop();
|
||||
if (!filename || filename.length < 20) continue;
|
||||
if (!seenFiles.has(filename)) {
|
||||
seenFiles.add(filename);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
console.log(`📋 Found ${docPages.length} pages with broader filter`);
|
||||
}
|
||||
|
||||
if (docPages.length === 0) {
|
||||
console.log("❌ Still no doc pages. Dumping first 30 link samples:");
|
||||
allLinks.slice(0, 30).forEach(l => console.log(" ", l.href));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Print a few
|
||||
docPages.slice(0, 5).forEach((p, i) => console.log(` ${i + 1}. ${p.href.split("/").pop().slice(0, 50)}`));
|
||||
console.log(` ... and ${docPages.length - Math.min(5, docPages.length)} more`);
|
||||
|
||||
// Convert each page to PDF
|
||||
const pdfs = [];
|
||||
for (let i = 0; i < docPages.length; i++) {
|
||||
const { href, text } = docPages[i];
|
||||
const fname = path.join(OUT_DIR, String(i).padStart(4, "0") + ".pdf");
|
||||
|
||||
if (fs.existsSync(fname) && fs.statSync(fname).size > 1000) {
|
||||
console.log(` ⏭ [${i+1}/${docPages.length}] skip: ${text.slice(0, 60)}`);
|
||||
pdfs.push(fname);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(` 📄 [${i+1}/${docPages.length}] ${text.slice(0, 60)}`);
|
||||
try {
|
||||
const p = await browser.newPage();
|
||||
await p.goto(href, { waitUntil: "networkidle2", timeout: 30000 });
|
||||
// Short wait for dynamic content
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
await p.pdf({ path: fname, format: "A4", printBackground: true });
|
||||
await p.close();
|
||||
pdfs.push(fname);
|
||||
} catch (e) {
|
||||
console.log(` ⚠️ Failed: ${e.message.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
if (pdfs.length > 0) {
|
||||
console.log(`\n📚 Merging ${pdfs.length} PDFs...`);
|
||||
execSync(`pdfunite ${pdfs.join(" ")} ${FINAL}`, { stdio: "pipe" });
|
||||
const stats = fs.statSync(FINAL);
|
||||
console.log(`✅ Done! ${FINAL} (${(stats.size/1024).toFixed(0)}KB, ${pdfs.length} pages)`);
|
||||
} else {
|
||||
console.log("❌ No PDFs generated");
|
||||
}
|
||||
})();
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
// Fast concurrent PDF crawler - converts multiple pages in parallel
|
||||
const puppeteer = require("puppeteer");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { execSync } = require("child_process");
|
||||
|
||||
const BASE = process.argv[2];
|
||||
const CONCURRENCY = parseInt(process.argv[3]) || 6;
|
||||
if (!BASE) { console.error("Usage: node crawl_fast.js <url> [concurrency]"); process.exit(1); }
|
||||
|
||||
const OUT_DIR = "/tmp/doc_pages";
|
||||
const FINAL = "/tmp/merged_guide.pdf";
|
||||
const baseUrl = new URL(BASE);
|
||||
const articleIdx = baseUrl.pathname.lastIndexOf("/");
|
||||
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
|
||||
|
||||
async function convertPage(browser, href, outPath, index, total) {
|
||||
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 500) return true;
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
|
||||
await page.pdf({ path: outPath, format: "A4", printBackground: true });
|
||||
await page.close();
|
||||
process.stdout.write(` [${index+1}/${total}]`);
|
||||
return true;
|
||||
} catch(e) {
|
||||
await page.close().catch(()=>{});
|
||||
process.stdout.write(` x`);
|
||||
fs.writeFileSync(outPath, ""); // marker for skip
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
fs.rmSync(OUT_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
|
||||
const mainPage = await browser.newPage();
|
||||
|
||||
console.log("🌐 Loading TOC:", BASE);
|
||||
await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
// Extract all doc links
|
||||
const allLinks = await mainPage.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
|
||||
});
|
||||
return links;
|
||||
});
|
||||
await mainPage.close();
|
||||
|
||||
// Filter: same article dir, en-us only, unique filenames
|
||||
const seenFiles = new Set();
|
||||
const docPages = [];
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
if (!u.pathname.startsWith("/en-us/")) continue;
|
||||
if (u.hash) continue;
|
||||
const fn = u.pathname.split("/").pop();
|
||||
if (!fn || fn.length < 20) continue;
|
||||
|
||||
// Skip obvious non-content pages
|
||||
const lower = link.text.toLowerCase();
|
||||
if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
|
||||
if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
|
||||
|
||||
if (!seenFiles.has(fn)) {
|
||||
seenFiles.add(fn);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
console.log(`📋 ${docPages.length} pages to convert with ${CONCURRENCY} concurrent tabs`);
|
||||
|
||||
if (docPages.length === 0) { console.log("❌ No pages found"); process.exit(1); }
|
||||
|
||||
const startTime = Date.now();
|
||||
const chunks = [];
|
||||
for (let i = 0; i < docPages.length; i += CONCURRENCY) {
|
||||
const batch = docPages.slice(i, i + CONCURRENCY);
|
||||
const tasks = batch.map((p, j) => {
|
||||
const out = path.join(OUT_DIR, String(i + j).padStart(4, "0") + ".pdf");
|
||||
chunks.push(out);
|
||||
return convertPage(browser, p.href, out, i + j, docPages.length);
|
||||
});
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
const validPdfs = chunks.filter(f => fs.existsSync(f) && fs.statSync(f).size > 500);
|
||||
const elapsed = ((Date.now() - startTime) / 1000).toFixed(0);
|
||||
console.log(`\n⏱ ${elapsed}s — ${validPdfs.length}/${docPages.length} pages converted`);
|
||||
|
||||
if (validPdfs.length > 0) {
|
||||
console.log(`📚 Merging...`);
|
||||
execSync(`pdfunite ${validPdfs.join(" ")} ${FINAL}`, { stdio: "pipe" });
|
||||
const stats = fs.statSync(FINAL);
|
||||
console.log(`✅ ${FINAL} (${(stats.size/1024/1024).toFixed(1)}MB)`);
|
||||
}
|
||||
})();
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
// Crawl Trend Micro docs and convert to a single Markdown file
|
||||
const puppeteer = require("puppeteer");
|
||||
const fs = require("fs");
|
||||
|
||||
const BASE = process.argv[2];
|
||||
const OUT = "/tmp/apex_one_admin_guide.md";
|
||||
const baseUrl = new URL(BASE);
|
||||
const articleIdx = baseUrl.pathname.lastIndexOf("/");
|
||||
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
|
||||
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
|
||||
const mainPage = await browser.newPage();
|
||||
|
||||
console.log("🌐 Loading TOC:", BASE);
|
||||
await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
// Extract all doc links
|
||||
const allLinks = await mainPage.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
|
||||
});
|
||||
return links;
|
||||
});
|
||||
await mainPage.close();
|
||||
|
||||
// Filter: same article dir, en-us only, unique filenames (same logic as PDF crawler)
|
||||
const seenFiles = new Set();
|
||||
const docPages = [];
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
if (!u.pathname.startsWith("/en-us/")) continue;
|
||||
if (u.hash) continue;
|
||||
const fn = u.pathname.split("/").pop();
|
||||
if (!fn || fn.length < 20) continue;
|
||||
const lower = link.text.toLowerCase();
|
||||
if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
|
||||
if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
|
||||
if (!seenFiles.has(fn)) {
|
||||
seenFiles.add(fn);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
console.log(`📋 ${docPages.length} pages to extract`);
|
||||
|
||||
const stream = fs.createWriteStream(OUT);
|
||||
stream.write("# Apex One Service Pack 1 (2025) — Admin Guide\n\n");
|
||||
stream.write(`> Crawled from ${BASE}\n\n`);
|
||||
stream.write("---\n\n");
|
||||
|
||||
for (let i = 0; i < docPages.length; i++) {
|
||||
const { href, text } = docPages[i];
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
// Extract title + body content
|
||||
const content = await page.evaluate(() => {
|
||||
// Try to find the main content area
|
||||
const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content", ".article-content"];
|
||||
let main = null;
|
||||
for (const sel of mainSel) {
|
||||
main = document.querySelector(sel);
|
||||
if (main) break;
|
||||
}
|
||||
if (!main) main = document.body;
|
||||
|
||||
// Get H1 first
|
||||
const h1 = main.querySelector("h1, h2");
|
||||
const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim();
|
||||
|
||||
// Get all text content, preserving some structure
|
||||
const blocks = [];
|
||||
main.querySelectorAll("h1, h2, h3, h4, p, li, table, pre, .note, .warning, .tip").forEach(el => {
|
||||
const tag = el.tagName;
|
||||
const txt = el.textContent.trim();
|
||||
if (!txt) return;
|
||||
if (tag.match(/^H[1-4]$/)) {
|
||||
const level = parseInt(tag[1]);
|
||||
blocks.push("\n" + "#".repeat(level + 1) + " " + txt + "\n");
|
||||
} else if (tag === "LI") {
|
||||
blocks.push("- " + txt);
|
||||
} else if (tag === "PRE") {
|
||||
blocks.push("\n```\n" + txt + "\n```\n");
|
||||
} else if (tag === "TABLE") {
|
||||
blocks.push("\n<!-- table omitted -->\n");
|
||||
} else {
|
||||
blocks.push(txt);
|
||||
}
|
||||
});
|
||||
|
||||
return { title, body: blocks.join("\n\n") };
|
||||
});
|
||||
|
||||
await page.close();
|
||||
|
||||
stream.write(`## ${i + 1}. ${content.title}\n\n`);
|
||||
stream.write(content.body || "_No content extracted_\n");
|
||||
stream.write("\n---\n\n");
|
||||
|
||||
if ((i + 1) % 50 === 0) console.log(` ${i + 1}/${docPages.length}`);
|
||||
} catch(e) {
|
||||
console.log(` ⚠️ [${i+1}] ${text.slice(0,40)}: ${e.message.slice(0,50)}`);
|
||||
stream.write(`## ${i + 1}. ${text}\n\n_Error: ${e.message}_\n\n---\n\n`);
|
||||
}
|
||||
}
|
||||
|
||||
stream.end();
|
||||
await browser.close();
|
||||
|
||||
const stats = fs.statSync(OUT);
|
||||
console.log(`\n✅ ${OUT} (${(stats.size/1024).toFixed(0)}KB)`);
|
||||
})();
|
||||
@@ -0,0 +1,120 @@
|
||||
// Batch crawl: process 100 pages, close/reopen browser to free memory
|
||||
const puppeteer = require("puppeteer");
|
||||
const fs = require("fs");
|
||||
|
||||
const BASE = process.argv[2];
|
||||
const START = parseInt(process.argv[3]) || 0;
|
||||
const BATCH = parseInt(process.argv[4]) || 100;
|
||||
const OUT = "/tmp/apex_one_admin_guide.md";
|
||||
const URLS_FILE = "/tmp/apex_one_urls.json";
|
||||
const baseUrl = new URL(BASE);
|
||||
const articleIdx = baseUrl.pathname.lastIndexOf("/");
|
||||
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);
|
||||
|
||||
async function getUrls() {
|
||||
if (fs.existsSync(URLS_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(URLS_FILE, "utf8"));
|
||||
}
|
||||
|
||||
const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
|
||||
const page = await browser.newPage();
|
||||
await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
const allLinks = await page.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
|
||||
});
|
||||
return links;
|
||||
});
|
||||
await browser.close();
|
||||
|
||||
const seenFiles = new Set();
|
||||
const docPages = [];
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
if (!u.pathname.startsWith("/en-us/")) continue;
|
||||
if (u.hash) continue;
|
||||
const fn = u.pathname.split("/").pop();
|
||||
if (!fn || fn.length < 20) continue;
|
||||
const lower = link.text.toLowerCase();
|
||||
if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
|
||||
if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
|
||||
if (!seenFiles.has(fn)) {
|
||||
seenFiles.add(fn);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
|
||||
return docPages;
|
||||
}
|
||||
|
||||
async function extractBatch(urls, start, count) {
|
||||
const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
|
||||
const end = Math.min(start + count, urls.length);
|
||||
|
||||
for (let i = start; i < end; i++) {
|
||||
const { href, text } = urls[i];
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
const content = await page.evaluate(() => {
|
||||
const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
|
||||
let main = document.body;
|
||||
for (const sel of mainSel) {
|
||||
const el = document.querySelector(sel);
|
||||
if (el) { main = el; break; }
|
||||
}
|
||||
const h1 = main.querySelector("h1, h2");
|
||||
const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim();
|
||||
|
||||
const blocks = [];
|
||||
main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
|
||||
const txt = el.textContent.trim();
|
||||
if (!txt || txt.length < 3) return;
|
||||
const tag = el.tagName;
|
||||
if (tag.match(/^H[1-4]$/)) {
|
||||
blocks.push("\n" + "#".repeat(parseInt(tag[1]) + 1) + " " + txt + "\n");
|
||||
} else if (tag === "LI") {
|
||||
blocks.push("- " + txt);
|
||||
} else if (tag === "PRE") {
|
||||
blocks.push("\n```\n" + txt + "\n```\n");
|
||||
} else {
|
||||
blocks.push(txt);
|
||||
}
|
||||
});
|
||||
return { title, body: blocks.join("\n\n") };
|
||||
});
|
||||
|
||||
await page.close();
|
||||
|
||||
const out = `## ${i + 1}. ${content.title}\n\n${content.body || "_no content_"} \n`;
|
||||
fs.appendFileSync(OUT, out);
|
||||
} catch(e) {
|
||||
fs.appendFileSync(OUT, `## ${i + 1}. ${text}\n\n_Error_ \n`);
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
console.log(`✅ [${start+1}-${end}/${urls.length}]`);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const urls = await getUrls();
|
||||
console.log(`📋 ${urls.length} pages total, starting at ${START}, batch ${BATCH}`);
|
||||
|
||||
// Write header if starting from 0
|
||||
if (START === 0) {
|
||||
fs.writeFileSync(OUT, `# Apex One Service Pack 1 (2025) — Admin Guide\n\n> ${urls.length} pages from ${BASE}\n\n`);
|
||||
}
|
||||
|
||||
await extractBatch(urls, START, BATCH);
|
||||
})();
|
||||
@@ -0,0 +1,44 @@
|
||||
const puppeteer = require("puppeteer");
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
||||
const page = await browser.newPage();
|
||||
|
||||
const base = process.argv[2];
|
||||
await page.goto(base, {waitUntil: "networkidle2", timeout: 30000});
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
const allLinks = await page.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
links.push({href: a.href, text: a.textContent.trim().slice(0, 60)});
|
||||
});
|
||||
return links;
|
||||
});
|
||||
|
||||
const baseUrl = new URL(base);
|
||||
const docsLinks = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname === baseUrl.hostname &&
|
||||
link.href.includes("/documentation/") &&
|
||||
!link.href.includes("#") &&
|
||||
!seen.has(link.href)) {
|
||||
seen.add(link.href);
|
||||
docsLinks.push(link);
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
console.log("Total links:", allLinks.length);
|
||||
console.log("Doc links:", docsLinks.length);
|
||||
|
||||
// Print them
|
||||
docsLinks.forEach((l, i) => {
|
||||
console.log(`${i+1}. ${l.href.split("/").pop().substring(0, 80)} — ${l.text}`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
Generated
+761
@@ -0,0 +1,761 @@
|
||||
{
|
||||
"name": "url2pdf",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "url2pdf",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"puppeteer": "^25.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
"version": "7.29.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz",
|
||||
"integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/helper-validator-identifier": "^7.28.5",
|
||||
"js-tokens": "^4.0.0",
|
||||
"picocolors": "^1.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-validator-identifier": {
|
||||
"version": "7.28.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz",
|
||||
"integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@puppeteer/browsers": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.3.tgz",
|
||||
"integrity": "sha512-v3YaiGpzUTgOZkHBFR0iZg58Vto25SqBQxfLUXDiofJccwVl6Mlr7BdLCS1NZgxikdeIHf936cxYWL9IZp3tow==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"debug": "^4.4.3",
|
||||
"progress": "^2.0.3",
|
||||
"semver": "^7.7.4",
|
||||
"tar-fs": "^3.1.1",
|
||||
"yargs": "^17.7.2"
|
||||
},
|
||||
"bin": {
|
||||
"browsers": "lib/main-cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.12.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"proxy-agent": ">=8.0.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"proxy-agent": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-regex": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
|
||||
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-convert": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/argparse": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/b4a": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz",
|
||||
"integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==",
|
||||
"license": "Apache-2.0",
|
||||
"peerDependencies": {
|
||||
"react-native-b4a": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"react-native-b4a": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/bare-events": {
|
||||
"version": "2.8.3",
|
||||
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.3.tgz",
|
||||
"integrity": "sha512-HdUm8EMQBLaJvGUdidNNbqpA1kYkwNcb+MYxkxCLAPJGQzlv9J0C24h8V65Z4c5GLd/JEALDvpFCQgpLJqc0zw==",
|
||||
"license": "Apache-2.0",
|
||||
"peerDependencies": {
|
||||
"bare-abort-controller": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bare-abort-controller": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/bare-fs": {
|
||||
"version": "4.7.1",
|
||||
"resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.1.tgz",
|
||||
"integrity": "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bare-events": "^2.5.4",
|
||||
"bare-path": "^3.0.0",
|
||||
"bare-stream": "^2.6.4",
|
||||
"bare-url": "^2.2.2",
|
||||
"fast-fifo": "^1.3.2"
|
||||
},
|
||||
"engines": {
|
||||
"bare": ">=1.16.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bare-buffer": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bare-buffer": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/bare-os": {
|
||||
"version": "3.9.1",
|
||||
"resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz",
|
||||
"integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"bare": ">=1.14.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bare-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz",
|
||||
"integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bare-os": "^3.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/bare-stream": {
|
||||
"version": "2.13.1",
|
||||
"resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.1.tgz",
|
||||
"integrity": "sha512-Vp0cnjYyrEC4whYTymQ+YZi6pBpfiICZO3cfRG8sy67ZNWe951urv1x4eW1BKNngw3U+3fPYb5JQvHbCtxH7Ow==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"streamx": "^2.25.0",
|
||||
"teex": "^1.0.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bare-abort-controller": "*",
|
||||
"bare-buffer": "*",
|
||||
"bare-events": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bare-abort-controller": {
|
||||
"optional": true
|
||||
},
|
||||
"bare-buffer": {
|
||||
"optional": true
|
||||
},
|
||||
"bare-events": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/bare-url": {
|
||||
"version": "2.4.3",
|
||||
"resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.3.tgz",
|
||||
"integrity": "sha512-Kccpc7ACfXaxfeInfqKcZtW4pT5YBn1mesc4sCsun6sRwtbJ4h+sNOaksUpYEJUKfN65YWC6Bw2OJEFiKxq8nQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bare-path": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/chromium-bidi": {
|
||||
"version": "16.0.1",
|
||||
"resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-16.0.1.tgz",
|
||||
"integrity": "sha512-J63PGu/9PpeCwLIcKYyzWP6yaVL5pxuBc0shlYCYM8BaAkmlwiQboXO1iNbOgSDbVklEyYFfNEcHD8oOAWacUA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"mitt": "^3.0.1",
|
||||
"zod": "^3.24.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.19.0 <22.0.0 || >=22.12.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"devtools-protocol": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/cliui": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
|
||||
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"string-width": "^4.2.0",
|
||||
"strip-ansi": "^6.0.1",
|
||||
"wrap-ansi": "^7.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-name": "~1.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cosmiconfig": {
|
||||
"version": "9.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.1.tgz",
|
||||
"integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"env-paths": "^2.2.1",
|
||||
"import-fresh": "^3.3.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"parse-json": "^5.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/d-fischer"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": ">=4.9.5"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"typescript": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/devtools-protocol": {
|
||||
"version": "0.0.1608973",
|
||||
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz",
|
||||
"integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/emoji-regex": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/end-of-stream": {
|
||||
"version": "1.4.5",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
|
||||
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/env-paths": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
|
||||
"integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/error-ex": {
|
||||
"version": "1.3.4",
|
||||
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz",
|
||||
"integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-arrayish": "^0.2.1"
|
||||
}
|
||||
},
|
||||
"node_modules/escalade": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/events-universal": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz",
|
||||
"integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bare-events": "^2.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-fifo": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
|
||||
"integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/get-caller-file": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": "6.* || 8.* || >= 10.*"
|
||||
}
|
||||
},
|
||||
"node_modules/import-fresh": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
||||
"integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"parent-module": "^1.0.0",
|
||||
"resolve-from": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-arrayish": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
|
||||
"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/js-tokens": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
|
||||
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/json-parse-even-better-errors": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
|
||||
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lines-and-columns": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
|
||||
"integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mitt": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
|
||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/parent-module": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
|
||||
"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"callsites": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/parse-json": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz",
|
||||
"integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.0.0",
|
||||
"error-ex": "^1.3.1",
|
||||
"json-parse-even-better-errors": "^2.3.0",
|
||||
"lines-and-columns": "^1.1.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
"integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/progress": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
|
||||
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pump": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
|
||||
"integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"end-of-stream": "^1.1.0",
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer": {
|
||||
"version": "25.0.4",
|
||||
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.0.4.tgz",
|
||||
"integrity": "sha512-QFdBAuNOqL0I+AdARTlRR1KcgPk0fo0dU127e1ZQFVxb9QPcpBDIiQp/dMgdbyLXHpF2GRjC/OezDmjKcLCKYw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@puppeteer/browsers": "3.0.3",
|
||||
"chromium-bidi": "16.0.1",
|
||||
"cosmiconfig": "^9.0.0",
|
||||
"devtools-protocol": "0.0.1608973",
|
||||
"puppeteer-core": "25.0.4",
|
||||
"typed-query-selector": "^2.12.2"
|
||||
},
|
||||
"bin": {
|
||||
"puppeteer": "lib/puppeteer/node/cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/puppeteer-core": {
|
||||
"version": "25.0.4",
|
||||
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.0.4.tgz",
|
||||
"integrity": "sha512-K1LQKDP6w1rIr1jUyN9obH16TO/DCy86k3q+FBd2prGY+TStxhFySxmaZZuRF+0D3BJXjwCYFke7tMHCH4olTA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@puppeteer/browsers": "3.0.3",
|
||||
"chromium-bidi": "16.0.1",
|
||||
"debug": "^4.4.3",
|
||||
"devtools-protocol": "0.0.1608973",
|
||||
"typed-query-selector": "^2.12.2",
|
||||
"webdriver-bidi-protocol": "0.4.1",
|
||||
"ws": "^8.20.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/resolve-from": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
|
||||
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.8.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz",
|
||||
"integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/streamx": {
|
||||
"version": "2.25.0",
|
||||
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz",
|
||||
"integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"events-universal": "^1.0.0",
|
||||
"fast-fifo": "^1.3.2",
|
||||
"text-decoder": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string-width": {
|
||||
"version": "4.2.3",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
|
||||
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"emoji-regex": "^8.0.0",
|
||||
"is-fullwidth-code-point": "^3.0.0",
|
||||
"strip-ansi": "^6.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-ansi": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
|
||||
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-regex": "^5.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/tar-fs": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz",
|
||||
"integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"pump": "^3.0.0",
|
||||
"tar-stream": "^3.1.5"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"bare-fs": "^4.0.1",
|
||||
"bare-path": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tar-stream": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz",
|
||||
"integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"b4a": "^1.6.4",
|
||||
"bare-fs": "^4.5.5",
|
||||
"fast-fifo": "^1.2.0",
|
||||
"streamx": "^2.15.0"
|
||||
}
|
||||
},
|
||||
"node_modules/teex": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz",
|
||||
"integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"streamx": "^2.12.5"
|
||||
}
|
||||
},
|
||||
"node_modules/text-decoder": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz",
|
||||
"integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"b4a": "^1.6.4"
|
||||
}
|
||||
},
|
||||
"node_modules/typed-query-selector": {
|
||||
"version": "2.12.2",
|
||||
"resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz",
|
||||
"integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/webdriver-bidi-protocol": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
||||
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/wrap-ansi": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.0.0",
|
||||
"string-width": "^4.1.0",
|
||||
"strip-ansi": "^6.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.20.1",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz",
|
||||
"integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/yargs": {
|
||||
"version": "17.7.2",
|
||||
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
|
||||
"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cliui": "^8.0.1",
|
||||
"escalade": "^3.1.1",
|
||||
"get-caller-file": "^2.0.5",
|
||||
"require-directory": "^2.1.1",
|
||||
"string-width": "^4.2.3",
|
||||
"y18n": "^5.0.5",
|
||||
"yargs-parser": "^21.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/yargs-parser": {
|
||||
"version": "21.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
|
||||
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "3.25.76",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
|
||||
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "url2pdf",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "pdf.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"puppeteer": "^25.0.4"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
const puppeteer = require("puppeteer");
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000});
|
||||
await page.pdf({path: "/tmp/page.pdf", format: "A4"});
|
||||
await browser.close();
|
||||
console.log("done");
|
||||
})();
|
||||
@@ -0,0 +1,455 @@
|
||||
#!/usr/bin/env python3
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
import subprocess, tempfile, os, json, urllib.parse, time, threading, shutil
|
||||
|
||||
JOBS_DIR = "/tmp/url2pdf_jobs"
|
||||
os.makedirs(JOBS_DIR, exist_ok=True)
|
||||
CWD = "/opt/url2pdf"
|
||||
|
||||
PUPPETEER_JS = """
|
||||
const puppeteer = require("puppeteer");
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000});
|
||||
await page.pdf({path: process.argv[3], format: "A4", printBackground: true});
|
||||
await browser.close();
|
||||
})();
|
||||
"""
|
||||
|
||||
CRAWLER_JS = """
|
||||
// Documentation crawler - cwd must have puppeteer installed
|
||||
const puppeteer = require("puppeteer");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { execSync } = require("child_process");
|
||||
|
||||
const BASE = process.argv[2];
|
||||
const JOB_DIR = process.argv[3];
|
||||
const baseUrl = new URL(BASE);
|
||||
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf("/") + 1);
|
||||
|
||||
function log(msg) {
|
||||
fs.appendFileSync(path.join(JOB_DIR, "log.txt"), msg + "\\n");
|
||||
process.stdout.write(msg + "\\n");
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const statusPath = path.join(JOB_DIR, "status.json");
|
||||
function setStatus(s) {
|
||||
fs.writeFileSync(statusPath, JSON.stringify({status: s, time: new Date().toISOString()}));
|
||||
}
|
||||
|
||||
try {
|
||||
setStatus("extracting_urls");
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
||||
const mainPage = await browser.newPage();
|
||||
|
||||
await mainPage.goto(BASE, {waitUntil: "networkidle2", timeout: 30000});
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
const allLinks = await mainPage.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) links.push({href: a.href, text: text.slice(0, 80)});
|
||||
});
|
||||
return links;
|
||||
});
|
||||
await mainPage.close();
|
||||
await browser.close();
|
||||
|
||||
// Filter unique doc pages (en-us only, dedupe by filename, skip language switchers)
|
||||
const seenFiles = new Set();
|
||||
const docPages = [];
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
if (!u.pathname.startsWith("/en-us/")) continue;
|
||||
if (u.hash) continue;
|
||||
const fn = u.pathname.split("/").pop();
|
||||
if (!fn || fn.length < 20) continue;
|
||||
const lower = link.text.toLowerCase();
|
||||
if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
|
||||
if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
|
||||
if (!seenFiles.has(fn)) {
|
||||
seenFiles.add(fn);
|
||||
docPages.push({href: link.href, text: link.text});
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
log("Found " + docPages.length + " pages");
|
||||
setStatus("crawling:" + docPages.length);
|
||||
|
||||
// Process in batches of 20 to avoid OOM
|
||||
const BATCH = 20;
|
||||
const mdPath = path.join(JOB_DIR, "output.md");
|
||||
const pdfDir = path.join(JOB_DIR, "pdfs");
|
||||
fs.mkdirSync(pdfDir, {recursive: true});
|
||||
|
||||
// Write MD header
|
||||
fs.writeFileSync(mdPath, "# " + baseUrl.hostname.split(".")[0] + " Documentation\\n\\n> " + docPages.length + " pages from " + BASE + "\\n\\n");
|
||||
|
||||
const allPdfs = [];
|
||||
|
||||
for (let batchStart = 0; batchStart < docPages.length; batchStart += BATCH) {
|
||||
const batchEnd = Math.min(batchStart + BATCH, docPages.length);
|
||||
setStatus("processing:" + batchStart + "/" + docPages.length);
|
||||
|
||||
const b = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
||||
|
||||
for (let i = batchStart; i < batchEnd; i++) {
|
||||
const {href, text} = docPages[i];
|
||||
try {
|
||||
const p = await b.newPage();
|
||||
await p.goto(href, {waitUntil: "networkidle2", timeout: 20000});
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
// Extract text for MD
|
||||
const content = await p.evaluate(() => {
|
||||
const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
|
||||
let main = document.body;
|
||||
for (const sel of mainSel) { const el = document.querySelector(sel); if (el) { main = el; break; } }
|
||||
const h1 = main.querySelector("h1, h2");
|
||||
const title = h1 ? h1.textContent.trim() : document.title.replace(/ \\| .*$/, "").trim();
|
||||
const blocks = [];
|
||||
main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
|
||||
const txt = el.textContent.trim();
|
||||
if (!txt || txt.length < 3) return;
|
||||
const tag = el.tagName;
|
||||
if (tag.match(/^H[1-4]$/)) blocks.push("\\n" + "#".repeat(parseInt(tag[1])+1) + " " + txt + "\\n");
|
||||
else if (tag === "LI") blocks.push("- " + txt);
|
||||
else if (tag === "PRE") blocks.push("\\n```\\n" + txt + "\\n```\\n");
|
||||
else blocks.push(txt);
|
||||
});
|
||||
return {title, body: blocks.join("\\n\\n")};
|
||||
});
|
||||
|
||||
fs.appendFileSync(mdPath, "## " + (i+1) + ". " + content.title + "\\n\\n" + (content.body || "_") + "\\n\\n");
|
||||
|
||||
// PDF for this page
|
||||
const pdfFile = path.join(pdfDir, String(i).padStart(4, "0") + ".pdf");
|
||||
await p.pdf({path: pdfFile, format: "A4", printBackground: true});
|
||||
allPdfs.push(pdfFile);
|
||||
await p.close();
|
||||
} catch(e) {
|
||||
fs.appendFileSync(mdPath, "## " + (i+1) + ". " + text + "\\n\\n_Error_\\n\\n");
|
||||
}
|
||||
}
|
||||
|
||||
await b.close();
|
||||
log("Batch " + (batchStart/BATCH + 1) + ": " + (batchStart+1) + "-" + batchEnd + " / " + docPages.length);
|
||||
}
|
||||
|
||||
// Merge PDFs
|
||||
setStatus("merging");
|
||||
const finalPdf = path.join(JOB_DIR, "output.pdf");
|
||||
if (allPdfs.length > 0) {
|
||||
execSync("pdfunite " + allPdfs.join(" ") + " " + finalPdf, {stdio: "pipe"});
|
||||
}
|
||||
|
||||
const pdfSize = fs.existsSync(finalPdf) ? fs.statSync(finalPdf).size : 0;
|
||||
setStatus("done:" + pdfSize + ":" + docPages.length);
|
||||
log("DONE. PDF: " + (pdfSize/1024/1024).toFixed(1) + "MB, MD: " + (fs.statSync(mdPath).size/1024).toFixed(0) + "KB");
|
||||
|
||||
} catch(e) {
|
||||
setStatus("error:" + e.message.slice(0, 200));
|
||||
log("ERROR: " + e.message);
|
||||
}
|
||||
})();
|
||||
"""
|
||||
|
||||
HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>URL to PDF</title>
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:-apple-system,system-ui,sans-serif;background:#09090b;color:#f0f0f4;min-height:100vh}
|
||||
.card{background:#131317;border:1px solid #2a2a33;border-radius:16px;padding:32px;max-width:560px;width:100%;margin:20px auto}
|
||||
h1{font-size:1.3rem;font-weight:700;margin-bottom:4px;letter-spacing:-0.3px}
|
||||
.sub{color:#88889a;font-size:0.85rem;margin-bottom:20px}
|
||||
input{width:100%;padding:12px 16px;border-radius:10px;border:1px solid #2a2a33;background:#09090b;color:#f0f0f4;font-size:0.9rem;font-family:inherit;margin-bottom:12px;transition:.2s}
|
||||
input:focus{outline:none;border-color:#8b5cf6;box-shadow:0 0 0 3px rgba(139,92,246,0.1)}
|
||||
button{width:100%;padding:12px;border-radius:10px;border:none;background:#8b5cf6;color:#fff;font-size:0.95rem;font-weight:600;cursor:pointer;font-family:inherit;transition:.2s}
|
||||
button:hover{background:#7c3aed}
|
||||
button:disabled{opacity:0.4;cursor:not-allowed}
|
||||
.tabs{display:flex;gap:8px;margin-bottom:20px}
|
||||
.tab{padding:8px 16px;border-radius:8px;cursor:pointer;background:#1a1a22;border:1px solid #2a2a33;color:#88889a;font-size:0.85rem;font-family:inherit;transition:.2s}
|
||||
.tab.active{background:#8b5cf6;border-color:#8b5cf6;color:#fff}
|
||||
.tab-content{display:none}
|
||||
.tab-content.active{display:block}
|
||||
.result{margin-top:16px;padding:12px 16px;border-radius:10px;font-size:0.85rem;text-align:center}
|
||||
.success{background:rgba(16,185,129,0.1);color:#34d399}
|
||||
.error{background:rgba(239,68,68,0.1);color:#f87171}
|
||||
.info{color:#88889a;font-size:0.75rem;margin-top:12px;text-align:center}
|
||||
.loader{display:none;text-align:center;padding:16px;color:#88889a}
|
||||
.loader.active{display:block}
|
||||
.spinner{width:24px;height:24px;border:2px solid #2a2a33;border-top-color:#8b5cf6;border-radius:50%;animation:spin .6s linear infinite;margin:0 auto 8px}
|
||||
@keyframes spin{to{transform:rotate(360deg)}}
|
||||
.progress{height:6px;background:#1a1a22;border-radius:3px;margin-top:12px;overflow:hidden}
|
||||
.progress-bar{height:100%;background:#8b5cf6;border-radius:3px;transition:width .3s;width:0}
|
||||
a.dl-btn{display:inline-block;padding:10px 16px;border-radius:10px;background:#8b5cf6;color:#fff;text-decoration:none;font-weight:600;margin:4px;font-size:0.85rem}
|
||||
a.dl-btn.green{background:#059669}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="card" style="margin-top:60px">
|
||||
<h1>PDF Tools</h1>
|
||||
<p class="sub">Convert URLs & documentation sites</p>
|
||||
|
||||
<div class="tabs">
|
||||
<button class="tab active" onclick="switchTab('single')">Single URL</button>
|
||||
<button class="tab" onclick="switchTab('crawl')">Doc Site Crawler</button>
|
||||
</div>
|
||||
|
||||
<!-- Single URL -->
|
||||
<div class="tab-content active" id="tab-single">
|
||||
<form id="form">
|
||||
<input id="url" type="url" placeholder="https://..." required autofocus>
|
||||
<button type="submit" id="btn">Convert to PDF</button>
|
||||
</form>
|
||||
<div class="loader" id="loader"><div class="spinner"></div>Converting...</div>
|
||||
<div id="result"></div>
|
||||
<p class="info">Powered by Chrome (Puppeteer)</p>
|
||||
</div>
|
||||
|
||||
<!-- Doc Crawler -->
|
||||
<div class="tab-content" id="tab-crawl">
|
||||
<p class="sub" style="margin-bottom:12px">Crawl an entire documentation site into a single PDF + Markdown file</p>
|
||||
<form id="crawl-form">
|
||||
<input id="crawl-url" type="url" placeholder="Paste any doc page URL..." required>
|
||||
<button type="submit" id="crawl-btn">Start Crawling</button>
|
||||
</form>
|
||||
<div class="loader" id="crawl-loader"><div class="spinner"></div><span id="crawl-status"></span></div>
|
||||
<div class="progress"><div class="progress-bar" id="crawl-progress" style="width:0"></div></div>
|
||||
<div id="crawl-result"></div>
|
||||
<p class="info">Each doc page becomes one PDF page. Large sites may take a few minutes.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function switchTab(name){
|
||||
document.querySelectorAll(".tab").forEach(t=>t.classList.remove("active"));
|
||||
document.querySelector(".tab[onclick*="+name+"]")?.classList.add("active");
|
||||
document.getElementById("tab-single").classList.toggle("active",name==="single");
|
||||
document.getElementById("tab-crawl").classList.toggle("active",name==="crawl");
|
||||
}
|
||||
|
||||
// Single URL
|
||||
document.getElementById("form").addEventListener("submit",async e=>{
|
||||
e.preventDefault();
|
||||
var btn=document.getElementById("btn"),loader=document.getElementById("loader"),res=document.getElementById("result"),url=document.getElementById("url").value.trim();
|
||||
if(!url)return;
|
||||
btn.disabled=true;loader.classList.add("active");res.innerHTML="";
|
||||
try{
|
||||
var r=await fetch("/url2pdf/convert",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
|
||||
if(!r.ok)throw new Error(await r.text());
|
||||
var blob=await r.blob();
|
||||
var a=document.createElement("a");a.href=URL.createObjectURL(blob);a.download="output.pdf";a.click();
|
||||
res.innerHTML='<div class="success">PDF downloaded!</div>';
|
||||
}catch(err){res.innerHTML='<div class="error">'+err.message+"</div>"}
|
||||
btn.disabled=false;loader.classList.remove("active");
|
||||
});
|
||||
|
||||
// Doc Crawler
|
||||
document.getElementById("crawl-form").addEventListener("submit",async e=>{
|
||||
e.preventDefault();
|
||||
var btn=document.getElementById("crawl-btn"),loader=document.getElementById("crawl-loader"),status=document.getElementById("crawl-status"),progress=document.getElementById("crawl-progress"),res=document.getElementById("crawl-result"),url=document.getElementById("crawl-url").value.trim();
|
||||
if(!url)return;
|
||||
btn.disabled=true;loader.classList.add("active");res.innerHTML="";progress.style.width="0";
|
||||
status.textContent="Starting crawl...";
|
||||
try{
|
||||
var r=await fetch("/url2pdf/crawl",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({url:url})});
|
||||
if(!r.ok)throw new Error(await r.text());
|
||||
var data=await r.json();
|
||||
var job=data.job;
|
||||
|
||||
// Poll for status
|
||||
var poll=setInterval(async()=>{
|
||||
try{
|
||||
var sr=await fetch("/url2pdf/crawl/status?job="+job);
|
||||
var s=await sr.json();
|
||||
if(s.status.startsWith("processing:")){
|
||||
var parts=s.status.split(":");
|
||||
var cur=parseInt(parts[1]),total=parseInt(parts[2]);
|
||||
var pct=Math.round((cur/total)*100);
|
||||
status.textContent="Processing page "+(cur+1)+" of "+total+" ("+pct+"%)";
|
||||
progress.style.width=pct+"%";
|
||||
}else if(s.status.startsWith("done:")){
|
||||
clearInterval(poll);
|
||||
var parts=s.status.split(":");
|
||||
status.textContent="Done! "+parts[2]+" pages";
|
||||
progress.style.width="100%";
|
||||
btn.disabled=false;loader.classList.remove("active");
|
||||
res.innerHTML='<a class="dl-btn" href="/url2pdf/crawl/download?job='+job+'&format=pdf">Download PDF</a> <a class="dl-btn green" href="/url2pdf/crawl/download?job='+job+'&format=md">Download Markdown</a>';
|
||||
}else if(s.status.startsWith("error:")){
|
||||
clearInterval(poll);
|
||||
status.textContent=s.status;
|
||||
btn.disabled=false;loader.classList.remove("active");
|
||||
res.innerHTML='<div class="error">'+s.status+"</div>";
|
||||
}else{
|
||||
status.textContent=s.status;
|
||||
}
|
||||
}catch(err){}
|
||||
},2000);
|
||||
}catch(err){
|
||||
btn.disabled=false;loader.classList.remove("active");
|
||||
res.innerHTML='<div class="error">'+err.message+"</div>";
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body></html>"""
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
pass # silence logs
|
||||
|
||||
def _json(self, code, data):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(data).encode())
|
||||
|
||||
def do_GET(self):
|
||||
if self.path.startswith("/crawl/status"):
|
||||
qs = urllib.parse.urlparse(self.path).query
|
||||
params = urllib.parse.parse_qs(qs)
|
||||
job = params.get("job", [None])[0]
|
||||
if not job:
|
||||
self._json(400, {"error": "Missing job"})
|
||||
return
|
||||
sp = os.path.join(JOBS_DIR, job, "status.json")
|
||||
if os.path.exists(sp):
|
||||
with open(sp) as f:
|
||||
self._json(200, json.load(f))
|
||||
else:
|
||||
self._json(200, {"status": "pending"})
|
||||
return
|
||||
|
||||
if self.path.startswith("/crawl/download"):
|
||||
qs = urllib.parse.urlparse(self.path).query
|
||||
params = urllib.parse.parse_qs(qs)
|
||||
job = params.get("job", [None])[0]
|
||||
fmt = params.get("format", ["pdf"])[0]
|
||||
if not job:
|
||||
self.send_error(400)
|
||||
return
|
||||
|
||||
jdir = os.path.join(JOBS_DIR, job)
|
||||
if fmt == "md":
|
||||
fp = os.path.join(jdir, "output.md")
|
||||
ct = "text/markdown"
|
||||
else:
|
||||
fp = os.path.join(jdir, "output.pdf")
|
||||
ct = "application/pdf"
|
||||
|
||||
if not os.path.exists(fp):
|
||||
self.send_error(404, "Not found")
|
||||
return
|
||||
|
||||
with open(fp, "rb") as f:
|
||||
data = f.read()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", ct)
|
||||
self.send_header("Content-Disposition", f'attachment; filename="{job}.{fmt}"')
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
return
|
||||
|
||||
# Serve HTML
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.end_headers()
|
||||
self.wfile.write(HTML.encode())
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except:
|
||||
self.send_error(400, "Bad JSON")
|
||||
return
|
||||
|
||||
if self.path == "/convert":
|
||||
url = data.get("url", "").strip()
|
||||
if not url:
|
||||
self.send_error(400, "No URL")
|
||||
return
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
self.send_error(400, "Invalid URL scheme")
|
||||
return
|
||||
|
||||
js_path = os.path.join(CWD, "convert.js")
|
||||
if not os.path.exists(js_path):
|
||||
with open(js_path, "w") as f:
|
||||
f.write(PUPPETEER_JS)
|
||||
|
||||
fd, tmp = tempfile.mkstemp(suffix=".pdf")
|
||||
os.close(fd)
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["timeout", "40", "node", js_path, url, tmp],
|
||||
capture_output=True, text=True, timeout=45, cwd=CWD
|
||||
)
|
||||
if r.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) < 100:
|
||||
raise Exception("Conversion failed: " + r.stderr[:200])
|
||||
|
||||
with open(tmp, "rb") as f:
|
||||
pdf_data = f.read()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/pdf")
|
||||
self.send_header("Content-Disposition", 'attachment; filename="output.pdf"')
|
||||
self.send_header("Content-Length", str(len(pdf_data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(pdf_data)
|
||||
except Exception as e:
|
||||
self.send_error(500, str(e))
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
|
||||
elif self.path == "/crawl":
|
||||
url = data.get("url", "").strip()
|
||||
if not url:
|
||||
self._json(400, {"error": "No URL"})
|
||||
return
|
||||
|
||||
import uuid
|
||||
job = uuid.uuid4().hex[:12]
|
||||
jdir = os.path.join(JOBS_DIR, job)
|
||||
os.makedirs(jdir, exist_ok=True)
|
||||
|
||||
# Write crawler script to npm dir so require() works
|
||||
crawl_js = os.path.join(CWD, f"crawler_{job}.js")
|
||||
with open(crawl_js, "w") as f:
|
||||
f.write(CRAWLER_JS)
|
||||
|
||||
# Write initial status
|
||||
with open(os.path.join(jdir, "status.json"), "w") as f:
|
||||
json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
|
||||
|
||||
# Start background crawler
|
||||
def run():
|
||||
subprocess.run(
|
||||
["timeout", "600", "node", crawl_js, url, jdir],
|
||||
capture_output=True, text=True, cwd=CWD,
|
||||
env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
|
||||
)
|
||||
# Clean up script
|
||||
try: os.unlink(crawl_js)
|
||||
except: pass
|
||||
|
||||
t = threading.Thread(target=run, daemon=True)
|
||||
t.start()
|
||||
|
||||
self._json(200, {"job": job, "status": "started"})
|
||||
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
HTTPServer(("127.0.0.1", 8099), Handler).serve_forever()
|
||||
Reference in New Issue
Block a user