v1.0.0: URL-to-PDF + Doc Site Crawler
- Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
const puppeteer = require("puppeteer");
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]});
|
||||
const page = await browser.newPage();
|
||||
|
||||
const base = process.argv[2];
|
||||
await page.goto(base, {waitUntil: "networkidle2", timeout: 30000});
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
const allLinks = await page.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll("a[href]").forEach(a => {
|
||||
links.push({href: a.href, text: a.textContent.trim().slice(0, 60)});
|
||||
});
|
||||
return links;
|
||||
});
|
||||
|
||||
const baseUrl = new URL(base);
|
||||
const docsLinks = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname === baseUrl.hostname &&
|
||||
link.href.includes("/documentation/") &&
|
||||
!link.href.includes("#") &&
|
||||
!seen.has(link.href)) {
|
||||
seen.add(link.href);
|
||||
docsLinks.push(link);
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
console.log("Total links:", allLinks.length);
|
||||
console.log("Doc links:", docsLinks.length);
|
||||
|
||||
// Print them
|
||||
docsLinks.forEach((l, i) => {
|
||||
console.log(`${i+1}. ${l.href.split("/").pop().substring(0, 80)} — ${l.text}`);
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
Reference in New Issue
Block a user