commit 0d3e40f797752fa4670428f65aafa17b6c273517 Author: IT Dog Date: Fri May 22 14:24:05 2026 +0800 v1.0.0: URL-to-PDF + Doc Site Crawler - Single URL to PDF via Puppeteer (Chrome headless) - Doc Site Crawler: crawl entire documentation sites - Auto-extract all pages from TOC navigation - 20-page batch processing with browser restart (anti-OOM) - Output: PDF + Markdown - Flask-style HTTP server on port 8099 - Deployed at https://pdf.donton.cloud/url2pdf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d8571f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +*.pdf +/tmp/ diff --git a/convert.js b/convert.js new file mode 100644 index 0000000..b3869d8 --- /dev/null +++ b/convert.js @@ -0,0 +1,9 @@ + +const puppeteer = require("puppeteer"); +(async () => { + const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]}); + const page = await browser.newPage(); + await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000}); + await page.pdf({path: process.argv[3], format: "A4", printBackground: true}); + await browser.close(); +})(); diff --git a/crawl.js b/crawl.js new file mode 100644 index 0000000..971e78d --- /dev/null +++ b/crawl.js @@ -0,0 +1,142 @@ +// Crawl a Trend Micro documentation site (or similar JS doc sites) +// Deduplicates language variants, filters out nav/UI links +const puppeteer = require("puppeteer"); +const fs = require("fs"); +const path = require("path"); +const { execSync } = require("child_process"); + +const BASE = process.argv[2]; +if (!BASE) { console.error("Usage: node crawl.js "); process.exit(1); } + +const OUT_DIR = "/tmp/doc_pages"; +const FINAL = "/tmp/merged_guide.pdf"; +const baseUrl = new URL(BASE); + +// Extract the documentation set prefix from URL +// e.g. /en-us/documentation/article/apex-one-service-pack-1-2025-server-online-help- +const articleIdx = baseUrl.pathname.lastIndexOf("/"); +const articleDir = baseUrl.pathname.substring(0, articleIdx + 1); + +(async () => { + fs.rmSync(OUT_DIR, { recursive: true, force: true }); + fs.mkdirSync(OUT_DIR, { recursive: true }); + + const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] }); + const page = await browser.newPage(); + + console.log("🌐 Loading:", BASE); + await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 }); + await new Promise(r => setTimeout(r, 3000)); + + // Extract ALL links from the fully loaded page + const allLinks = await page.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) { + links.push({ href: a.href, text: text.slice(0, 80) }); + } + }); + return links; + }); + + // Filter: same host, under same article directory, not a hash link + // Also: unique by page filename (dedupe language variants) + const seenFiles = new Set(); + const docPages = []; + + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (u.hash) continue; + + const pathname = u.pathname; + // Must be under the same article directory + if (!pathname.startsWith(articleDir)) continue; + + // Get the page filename + const filename = pathname.split("/").pop(); + // Must match the doc page pattern (long hyphenated names ending with article ID) + if (!filename || filename.length < 20) continue; + + // Skip language-switch duplicates (same filename, different language path) + // The paths are like: /en-us/..., /fr-fr/..., /de-de/..., etc. + // Filter to only English (en-us) pages + if (!pathname.startsWith("/en-us/")) continue; + + if (!seenFiles.has(filename)) { + seenFiles.add(filename); + docPages.push({ href: link.href, text: link.text }); + } + } catch (e) {} + } + + console.log(`📋 Found ${docPages.length} unique English doc pages`); + if (docPages.length === 0) { + console.log("⚠️ No pages found with strict filter. Trying broader filter..."); + // Retry with just articleDir prefix, no language filter + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (u.hash) continue; + if (!u.pathname.startsWith(articleDir)) continue; + const filename = u.pathname.split("/").pop(); + if (!filename || filename.length < 20) continue; + if (!seenFiles.has(filename)) { + seenFiles.add(filename); + docPages.push({ href: link.href, text: link.text }); + } + } catch (e) {} + } + console.log(`📋 Found ${docPages.length} pages with broader filter`); + } + + if (docPages.length === 0) { + console.log("❌ Still no doc pages. Dumping first 30 link samples:"); + allLinks.slice(0, 30).forEach(l => console.log(" ", l.href)); + process.exit(1); + } + + // Print a few + docPages.slice(0, 5).forEach((p, i) => console.log(` ${i + 1}. ${p.href.split("/").pop().slice(0, 50)}`)); + console.log(` ... and ${docPages.length - Math.min(5, docPages.length)} more`); + + // Convert each page to PDF + const pdfs = []; + for (let i = 0; i < docPages.length; i++) { + const { href, text } = docPages[i]; + const fname = path.join(OUT_DIR, String(i).padStart(4, "0") + ".pdf"); + + if (fs.existsSync(fname) && fs.statSync(fname).size > 1000) { + console.log(` ⏭ [${i+1}/${docPages.length}] skip: ${text.slice(0, 60)}`); + pdfs.push(fname); + continue; + } + + console.log(` 📄 [${i+1}/${docPages.length}] ${text.slice(0, 60)}`); + try { + const p = await browser.newPage(); + await p.goto(href, { waitUntil: "networkidle2", timeout: 30000 }); + // Short wait for dynamic content + await new Promise(r => setTimeout(r, 1000)); + await p.pdf({ path: fname, format: "A4", printBackground: true }); + await p.close(); + pdfs.push(fname); + } catch (e) { + console.log(` ⚠️ Failed: ${e.message.slice(0, 80)}`); + } + } + + await browser.close(); + + if (pdfs.length > 0) { + console.log(`\n📚 Merging ${pdfs.length} PDFs...`); + execSync(`pdfunite ${pdfs.join(" ")} ${FINAL}`, { stdio: "pipe" }); + const stats = fs.statSync(FINAL); + console.log(`✅ Done! ${FINAL} (${(stats.size/1024).toFixed(0)}KB, ${pdfs.length} pages)`); + } else { + console.log("❌ No PDFs generated"); + } +})(); diff --git a/crawl_fast.js b/crawl_fast.js new file mode 100644 index 0000000..f808053 --- /dev/null +++ b/crawl_fast.js @@ -0,0 +1,109 @@ +// Fast concurrent PDF crawler - converts multiple pages in parallel +const puppeteer = require("puppeteer"); +const fs = require("fs"); +const path = require("path"); +const { execSync } = require("child_process"); + +const BASE = process.argv[2]; +const CONCURRENCY = parseInt(process.argv[3]) || 6; +if (!BASE) { console.error("Usage: node crawl_fast.js [concurrency]"); process.exit(1); } + +const OUT_DIR = "/tmp/doc_pages"; +const FINAL = "/tmp/merged_guide.pdf"; +const baseUrl = new URL(BASE); +const articleIdx = baseUrl.pathname.lastIndexOf("/"); +const articleDir = baseUrl.pathname.substring(0, articleIdx + 1); + +async function convertPage(browser, href, outPath, index, total) { + if (fs.existsSync(outPath) && fs.statSync(outPath).size > 500) return true; + const page = await browser.newPage(); + try { + await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 }); + await page.pdf({ path: outPath, format: "A4", printBackground: true }); + await page.close(); + process.stdout.write(` [${index+1}/${total}]`); + return true; + } catch(e) { + await page.close().catch(()=>{}); + process.stdout.write(` x`); + fs.writeFileSync(outPath, ""); // marker for skip + return false; + } +} + +(async () => { + fs.rmSync(OUT_DIR, { recursive: true, force: true }); + fs.mkdirSync(OUT_DIR, { recursive: true }); + + const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] }); + const mainPage = await browser.newPage(); + + console.log("🌐 Loading TOC:", BASE); + await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 }); + await new Promise(r => setTimeout(r, 3000)); + + // Extract all doc links + const allLinks = await mainPage.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) }); + }); + return links; + }); + await mainPage.close(); + + // Filter: same article dir, en-us only, unique filenames + const seenFiles = new Set(); + const docPages = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (!u.pathname.startsWith("/en-us/")) continue; + if (u.hash) continue; + const fn = u.pathname.split("/").pop(); + if (!fn || fn.length < 20) continue; + + // Skip obvious non-content pages + const lower = link.text.toLowerCase(); + if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue; + if (/online help center|security for endpoints|apex one$/i.test(lower)) continue; + + if (!seenFiles.has(fn)) { + seenFiles.add(fn); + docPages.push({ href: link.href, text: link.text }); + } + } catch(e) {} + } + + console.log(`📋 ${docPages.length} pages to convert with ${CONCURRENCY} concurrent tabs`); + + if (docPages.length === 0) { console.log("❌ No pages found"); process.exit(1); } + + const startTime = Date.now(); + const chunks = []; + for (let i = 0; i < docPages.length; i += CONCURRENCY) { + const batch = docPages.slice(i, i + CONCURRENCY); + const tasks = batch.map((p, j) => { + const out = path.join(OUT_DIR, String(i + j).padStart(4, "0") + ".pdf"); + chunks.push(out); + return convertPage(browser, p.href, out, i + j, docPages.length); + }); + await Promise.all(tasks); + } + + await browser.close(); + + const validPdfs = chunks.filter(f => fs.existsSync(f) && fs.statSync(f).size > 500); + const elapsed = ((Date.now() - startTime) / 1000).toFixed(0); + console.log(`\n⏱ ${elapsed}s — ${validPdfs.length}/${docPages.length} pages converted`); + + if (validPdfs.length > 0) { + console.log(`📚 Merging...`); + execSync(`pdfunite ${validPdfs.join(" ")} ${FINAL}`, { stdio: "pipe" }); + const stats = fs.statSync(FINAL); + console.log(`✅ ${FINAL} (${(stats.size/1024/1024).toFixed(1)}MB)`); + } +})(); diff --git a/crawl_md.js b/crawl_md.js new file mode 100644 index 0000000..27f510d --- /dev/null +++ b/crawl_md.js @@ -0,0 +1,122 @@ +// Crawl Trend Micro docs and convert to a single Markdown file +const puppeteer = require("puppeteer"); +const fs = require("fs"); + +const BASE = process.argv[2]; +const OUT = "/tmp/apex_one_admin_guide.md"; +const baseUrl = new URL(BASE); +const articleIdx = baseUrl.pathname.lastIndexOf("/"); +const articleDir = baseUrl.pathname.substring(0, articleIdx + 1); + +(async () => { + const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] }); + const mainPage = await browser.newPage(); + + console.log("🌐 Loading TOC:", BASE); + await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 }); + await new Promise(r => setTimeout(r, 3000)); + + // Extract all doc links + const allLinks = await mainPage.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) }); + }); + return links; + }); + await mainPage.close(); + + // Filter: same article dir, en-us only, unique filenames (same logic as PDF crawler) + const seenFiles = new Set(); + const docPages = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (!u.pathname.startsWith("/en-us/")) continue; + if (u.hash) continue; + const fn = u.pathname.split("/").pop(); + if (!fn || fn.length < 20) continue; + const lower = link.text.toLowerCase(); + if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue; + if (/online help center|security for endpoints|apex one$/i.test(lower)) continue; + if (!seenFiles.has(fn)) { + seenFiles.add(fn); + docPages.push({ href: link.href, text: link.text }); + } + } catch(e) {} + } + + console.log(`📋 ${docPages.length} pages to extract`); + + const stream = fs.createWriteStream(OUT); + stream.write("# Apex One Service Pack 1 (2025) — Admin Guide\n\n"); + stream.write(`> Crawled from ${BASE}\n\n`); + stream.write("---\n\n"); + + for (let i = 0; i < docPages.length; i++) { + const { href, text } = docPages[i]; + try { + const page = await browser.newPage(); + await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 }); + await new Promise(r => setTimeout(r, 500)); + + // Extract title + body content + const content = await page.evaluate(() => { + // Try to find the main content area + const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content", ".article-content"]; + let main = null; + for (const sel of mainSel) { + main = document.querySelector(sel); + if (main) break; + } + if (!main) main = document.body; + + // Get H1 first + const h1 = main.querySelector("h1, h2"); + const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim(); + + // Get all text content, preserving some structure + const blocks = []; + main.querySelectorAll("h1, h2, h3, h4, p, li, table, pre, .note, .warning, .tip").forEach(el => { + const tag = el.tagName; + const txt = el.textContent.trim(); + if (!txt) return; + if (tag.match(/^H[1-4]$/)) { + const level = parseInt(tag[1]); + blocks.push("\n" + "#".repeat(level + 1) + " " + txt + "\n"); + } else if (tag === "LI") { + blocks.push("- " + txt); + } else if (tag === "PRE") { + blocks.push("\n```\n" + txt + "\n```\n"); + } else if (tag === "TABLE") { + blocks.push("\n\n"); + } else { + blocks.push(txt); + } + }); + + return { title, body: blocks.join("\n\n") }; + }); + + await page.close(); + + stream.write(`## ${i + 1}. ${content.title}\n\n`); + stream.write(content.body || "_No content extracted_\n"); + stream.write("\n---\n\n"); + + if ((i + 1) % 50 === 0) console.log(` ${i + 1}/${docPages.length}`); + } catch(e) { + console.log(` ⚠️ [${i+1}] ${text.slice(0,40)}: ${e.message.slice(0,50)}`); + stream.write(`## ${i + 1}. ${text}\n\n_Error: ${e.message}_\n\n---\n\n`); + } + } + + stream.end(); + await browser.close(); + + const stats = fs.statSync(OUT); + console.log(`\n✅ ${OUT} (${(stats.size/1024).toFixed(0)}KB)`); +})(); diff --git a/crawl_md_batch.js b/crawl_md_batch.js new file mode 100644 index 0000000..6171ac8 --- /dev/null +++ b/crawl_md_batch.js @@ -0,0 +1,120 @@ +// Batch crawl: process 100 pages, close/reopen browser to free memory +const puppeteer = require("puppeteer"); +const fs = require("fs"); + +const BASE = process.argv[2]; +const START = parseInt(process.argv[3]) || 0; +const BATCH = parseInt(process.argv[4]) || 100; +const OUT = "/tmp/apex_one_admin_guide.md"; +const URLS_FILE = "/tmp/apex_one_urls.json"; +const baseUrl = new URL(BASE); +const articleIdx = baseUrl.pathname.lastIndexOf("/"); +const articleDir = baseUrl.pathname.substring(0, articleIdx + 1); + +async function getUrls() { + if (fs.existsSync(URLS_FILE)) { + return JSON.parse(fs.readFileSync(URLS_FILE, "utf8")); + } + + const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] }); + const page = await browser.newPage(); + await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 }); + await new Promise(r => setTimeout(r, 3000)); + + const allLinks = await page.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) }); + }); + return links; + }); + await browser.close(); + + const seenFiles = new Set(); + const docPages = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (!u.pathname.startsWith("/en-us/")) continue; + if (u.hash) continue; + const fn = u.pathname.split("/").pop(); + if (!fn || fn.length < 20) continue; + const lower = link.text.toLowerCase(); + if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue; + if (/online help center|security for endpoints|apex one$/i.test(lower)) continue; + if (!seenFiles.has(fn)) { + seenFiles.add(fn); + docPages.push({ href: link.href, text: link.text }); + } + } catch(e) {} + } + fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2)); + return docPages; +} + +async function extractBatch(urls, start, count) { + const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] }); + const end = Math.min(start + count, urls.length); + + for (let i = start; i < end; i++) { + const { href, text } = urls[i]; + try { + const page = await browser.newPage(); + await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 }); + await new Promise(r => setTimeout(r, 300)); + + const content = await page.evaluate(() => { + const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"]; + let main = document.body; + for (const sel of mainSel) { + const el = document.querySelector(sel); + if (el) { main = el; break; } + } + const h1 = main.querySelector("h1, h2"); + const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim(); + + const blocks = []; + main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => { + const txt = el.textContent.trim(); + if (!txt || txt.length < 3) return; + const tag = el.tagName; + if (tag.match(/^H[1-4]$/)) { + blocks.push("\n" + "#".repeat(parseInt(tag[1]) + 1) + " " + txt + "\n"); + } else if (tag === "LI") { + blocks.push("- " + txt); + } else if (tag === "PRE") { + blocks.push("\n```\n" + txt + "\n```\n"); + } else { + blocks.push(txt); + } + }); + return { title, body: blocks.join("\n\n") }; + }); + + await page.close(); + + const out = `## ${i + 1}. ${content.title}\n\n${content.body || "_no content_"} \n`; + fs.appendFileSync(OUT, out); + } catch(e) { + fs.appendFileSync(OUT, `## ${i + 1}. ${text}\n\n_Error_ \n`); + } + } + + await browser.close(); + console.log(`✅ [${start+1}-${end}/${urls.length}]`); +} + +(async () => { + const urls = await getUrls(); + console.log(`📋 ${urls.length} pages total, starting at ${START}, batch ${BATCH}`); + + // Write header if starting from 0 + if (START === 0) { + fs.writeFileSync(OUT, `# Apex One Service Pack 1 (2025) — Admin Guide\n\n> ${urls.length} pages from ${BASE}\n\n`); + } + + await extractBatch(urls, START, BATCH); +})(); diff --git a/find_links.js b/find_links.js new file mode 100644 index 0000000..9f0eb72 --- /dev/null +++ b/find_links.js @@ -0,0 +1,44 @@ +const puppeteer = require("puppeteer"); +(async () => { + const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); + const page = await browser.newPage(); + + const base = process.argv[2]; + await page.goto(base, {waitUntil: "networkidle2", timeout: 30000}); + await new Promise(r => setTimeout(r, 3000)); + + const allLinks = await page.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + links.push({href: a.href, text: a.textContent.trim().slice(0, 60)}); + }); + return links; + }); + + const baseUrl = new URL(base); + const docsLinks = []; + const seen = new Set(); + + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname === baseUrl.hostname && + link.href.includes("/documentation/") && + !link.href.includes("#") && + !seen.has(link.href)) { + seen.add(link.href); + docsLinks.push(link); + } + } catch(e) {} + } + + console.log("Total links:", allLinks.length); + console.log("Doc links:", docsLinks.length); + + // Print them + docsLinks.forEach((l, i) => { + console.log(`${i+1}. ${l.href.split("/").pop().substring(0, 80)} — ${l.text}`); + }); + + await browser.close(); +})(); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..74e9dee --- /dev/null +++ b/package-lock.json @@ -0,0 +1,761 @@ +{ + "name": "url2pdf", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "url2pdf", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "puppeteer": "^25.0.4" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.3.tgz", + "integrity": "sha512-v3YaiGpzUTgOZkHBFR0iZg58Vto25SqBQxfLUXDiofJccwVl6Mlr7BdLCS1NZgxikdeIHf936cxYWL9IZp3tow==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3", + "progress": "^2.0.3", + "semver": "^7.7.4", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/bare-events": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.3.tgz", + "integrity": "sha512-HdUm8EMQBLaJvGUdidNNbqpA1kYkwNcb+MYxkxCLAPJGQzlv9J0C24h8V65Z4c5GLd/JEALDvpFCQgpLJqc0zw==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.1.tgz", + "integrity": "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz", + "integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.1.tgz", + "integrity": "sha512-Vp0cnjYyrEC4whYTymQ+YZi6pBpfiICZO3cfRG8sy67ZNWe951urv1x4eW1BKNngw3U+3fPYb5JQvHbCtxH7Ow==", + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.3.tgz", + "integrity": "sha512-Kccpc7ACfXaxfeInfqKcZtW4pT5YBn1mesc4sCsun6sRwtbJ4h+sNOaksUpYEJUKfN65YWC6Bw2OJEFiKxq8nQ==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chromium-bidi": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-16.0.1.tgz", + "integrity": "sha512-J63PGu/9PpeCwLIcKYyzWP6yaVL5pxuBc0shlYCYM8BaAkmlwiQboXO1iNbOgSDbVklEyYFfNEcHD8oOAWacUA==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.1.tgz", + "integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "license": "BSD-3-Clause" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer": { + "version": "25.0.4", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-25.0.4.tgz", + "integrity": "sha512-QFdBAuNOqL0I+AdARTlRR1KcgPk0fo0dU127e1ZQFVxb9QPcpBDIiQp/dMgdbyLXHpF2GRjC/OezDmjKcLCKYw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.0.3", + "chromium-bidi": "16.0.1", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1608973", + "puppeteer-core": "25.0.4", + "typed-query-selector": "^2.12.2" + }, + "bin": { + "puppeteer": "lib/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/puppeteer-core": { + "version": "25.0.4", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.0.4.tgz", + "integrity": "sha512-K1LQKDP6w1rIr1jUyN9obH16TO/DCy86k3q+FBd2prGY+TStxhFySxmaZZuRF+0D3BJXjwCYFke7tMHCH4olTA==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.0.3", + "chromium-bidi": "16.0.1", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/semver": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz", + "integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/streamx": { + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "license": "MIT" + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "license": "Apache-2.0" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.20.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz", + "integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c7d9b31 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "url2pdf", + "version": "1.0.0", + "description": "", + "main": "pdf.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "puppeteer": "^25.0.4" + } +} diff --git a/pdf.js b/pdf.js new file mode 100644 index 0000000..4dbcf75 --- /dev/null +++ b/pdf.js @@ -0,0 +1,9 @@ +const puppeteer = require("puppeteer"); +(async () => { + const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); + const page = await browser.newPage(); + await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000}); + await page.pdf({path: "/tmp/page.pdf", format: "A4"}); + await browser.close(); + console.log("done"); +})(); diff --git a/server.py b/server.py new file mode 100644 index 0000000..1942584 --- /dev/null +++ b/server.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +from http.server import HTTPServer, BaseHTTPRequestHandler +import subprocess, tempfile, os, json, urllib.parse, time, threading, shutil + +JOBS_DIR = "/tmp/url2pdf_jobs" +os.makedirs(JOBS_DIR, exist_ok=True) +CWD = "/opt/url2pdf" + +PUPPETEER_JS = """ +const puppeteer = require("puppeteer"); +(async () => { + const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox", "--disable-gpu"]}); + const page = await browser.newPage(); + await page.goto(process.argv[2], {waitUntil: "networkidle2", timeout: 30000}); + await page.pdf({path: process.argv[3], format: "A4", printBackground: true}); + await browser.close(); +})(); +""" + +CRAWLER_JS = """ +// Documentation crawler - cwd must have puppeteer installed +const puppeteer = require("puppeteer"); +const fs = require("fs"); +const path = require("path"); +const { execSync } = require("child_process"); + +const BASE = process.argv[2]; +const JOB_DIR = process.argv[3]; +const baseUrl = new URL(BASE); +const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf("/") + 1); + +function log(msg) { + fs.appendFileSync(path.join(JOB_DIR, "log.txt"), msg + "\\n"); + process.stdout.write(msg + "\\n"); +} + +(async () => { + const statusPath = path.join(JOB_DIR, "status.json"); + function setStatus(s) { + fs.writeFileSync(statusPath, JSON.stringify({status: s, time: new Date().toISOString()})); + } + + try { + setStatus("extracting_urls"); + const browser = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); + const mainPage = await browser.newPage(); + + await mainPage.goto(BASE, {waitUntil: "networkidle2", timeout: 30000}); + await new Promise(r => setTimeout(r, 3000)); + + const allLinks = await mainPage.evaluate(() => { + const links = []; + document.querySelectorAll("a[href]").forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({href: a.href, text: text.slice(0, 80)}); + }); + return links; + }); + await mainPage.close(); + await browser.close(); + + // Filter unique doc pages (en-us only, dedupe by filename, skip language switchers) + const seenFiles = new Set(); + const docPages = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (!u.pathname.startsWith("/en-us/")) continue; + if (u.hash) continue; + const fn = u.pathname.split("/").pop(); + if (!fn || fn.length < 20) continue; + const lower = link.text.toLowerCase(); + if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue; + if (/online help center|security for endpoints|apex one$/i.test(lower)) continue; + if (!seenFiles.has(fn)) { + seenFiles.add(fn); + docPages.push({href: link.href, text: link.text}); + } + } catch(e) {} + } + + log("Found " + docPages.length + " pages"); + setStatus("crawling:" + docPages.length); + + // Process in batches of 20 to avoid OOM + const BATCH = 20; + const mdPath = path.join(JOB_DIR, "output.md"); + const pdfDir = path.join(JOB_DIR, "pdfs"); + fs.mkdirSync(pdfDir, {recursive: true}); + + // Write MD header + fs.writeFileSync(mdPath, "# " + baseUrl.hostname.split(".")[0] + " Documentation\\n\\n> " + docPages.length + " pages from " + BASE + "\\n\\n"); + + const allPdfs = []; + + for (let batchStart = 0; batchStart < docPages.length; batchStart += BATCH) { + const batchEnd = Math.min(batchStart + BATCH, docPages.length); + setStatus("processing:" + batchStart + "/" + docPages.length); + + const b = await puppeteer.launch({headless: true, args: ["--no-sandbox"]}); + + for (let i = batchStart; i < batchEnd; i++) { + const {href, text} = docPages[i]; + try { + const p = await b.newPage(); + await p.goto(href, {waitUntil: "networkidle2", timeout: 20000}); + await new Promise(r => setTimeout(r, 300)); + + // Extract text for MD + const content = await p.evaluate(() => { + const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"]; + let main = document.body; + for (const sel of mainSel) { const el = document.querySelector(sel); if (el) { main = el; break; } } + const h1 = main.querySelector("h1, h2"); + const title = h1 ? h1.textContent.trim() : document.title.replace(/ \\| .*$/, "").trim(); + const blocks = []; + main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => { + const txt = el.textContent.trim(); + if (!txt || txt.length < 3) return; + const tag = el.tagName; + if (tag.match(/^H[1-4]$/)) blocks.push("\\n" + "#".repeat(parseInt(tag[1])+1) + " " + txt + "\\n"); + else if (tag === "LI") blocks.push("- " + txt); + else if (tag === "PRE") blocks.push("\\n```\\n" + txt + "\\n```\\n"); + else blocks.push(txt); + }); + return {title, body: blocks.join("\\n\\n")}; + }); + + fs.appendFileSync(mdPath, "## " + (i+1) + ". " + content.title + "\\n\\n" + (content.body || "_") + "\\n\\n"); + + // PDF for this page + const pdfFile = path.join(pdfDir, String(i).padStart(4, "0") + ".pdf"); + await p.pdf({path: pdfFile, format: "A4", printBackground: true}); + allPdfs.push(pdfFile); + await p.close(); + } catch(e) { + fs.appendFileSync(mdPath, "## " + (i+1) + ". " + text + "\\n\\n_Error_\\n\\n"); + } + } + + await b.close(); + log("Batch " + (batchStart/BATCH + 1) + ": " + (batchStart+1) + "-" + batchEnd + " / " + docPages.length); + } + + // Merge PDFs + setStatus("merging"); + const finalPdf = path.join(JOB_DIR, "output.pdf"); + if (allPdfs.length > 0) { + execSync("pdfunite " + allPdfs.join(" ") + " " + finalPdf, {stdio: "pipe"}); + } + + const pdfSize = fs.existsSync(finalPdf) ? fs.statSync(finalPdf).size : 0; + setStatus("done:" + pdfSize + ":" + docPages.length); + log("DONE. PDF: " + (pdfSize/1024/1024).toFixed(1) + "MB, MD: " + (fs.statSync(mdPath).size/1024).toFixed(0) + "KB"); + + } catch(e) { + setStatus("error:" + e.message.slice(0, 200)); + log("ERROR: " + e.message); + } +})(); +""" + +HTML = """ + +URL to PDF + + +
+

PDF Tools

+

Convert URLs & documentation sites

+ +
+ + +
+ + +
+
+ + +
+
Converting...
+
+

Powered by Chrome (Puppeteer)

+
+ + +
+

Crawl an entire documentation site into a single PDF + Markdown file

+
+ + +
+
+
+
+

Each doc page becomes one PDF page. Large sites may take a few minutes.

+
+
+ + +""" + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass # silence logs + + def _json(self, code, data): + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + def do_GET(self): + if self.path.startswith("/crawl/status"): + qs = urllib.parse.urlparse(self.path).query + params = urllib.parse.parse_qs(qs) + job = params.get("job", [None])[0] + if not job: + self._json(400, {"error": "Missing job"}) + return + sp = os.path.join(JOBS_DIR, job, "status.json") + if os.path.exists(sp): + with open(sp) as f: + self._json(200, json.load(f)) + else: + self._json(200, {"status": "pending"}) + return + + if self.path.startswith("/crawl/download"): + qs = urllib.parse.urlparse(self.path).query + params = urllib.parse.parse_qs(qs) + job = params.get("job", [None])[0] + fmt = params.get("format", ["pdf"])[0] + if not job: + self.send_error(400) + return + + jdir = os.path.join(JOBS_DIR, job) + if fmt == "md": + fp = os.path.join(jdir, "output.md") + ct = "text/markdown" + else: + fp = os.path.join(jdir, "output.pdf") + ct = "application/pdf" + + if not os.path.exists(fp): + self.send_error(404, "Not found") + return + + with open(fp, "rb") as f: + data = f.read() + self.send_response(200) + self.send_header("Content-Type", ct) + self.send_header("Content-Disposition", f'attachment; filename="{job}.{fmt}"') + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + + # Serve HTML + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(HTML.encode()) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + except: + self.send_error(400, "Bad JSON") + return + + if self.path == "/convert": + url = data.get("url", "").strip() + if not url: + self.send_error(400, "No URL") + return + parsed = urllib.parse.urlparse(url) + if parsed.scheme not in ("http", "https"): + self.send_error(400, "Invalid URL scheme") + return + + js_path = os.path.join(CWD, "convert.js") + if not os.path.exists(js_path): + with open(js_path, "w") as f: + f.write(PUPPETEER_JS) + + fd, tmp = tempfile.mkstemp(suffix=".pdf") + os.close(fd) + try: + r = subprocess.run( + ["timeout", "40", "node", js_path, url, tmp], + capture_output=True, text=True, timeout=45, cwd=CWD + ) + if r.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) < 100: + raise Exception("Conversion failed: " + r.stderr[:200]) + + with open(tmp, "rb") as f: + pdf_data = f.read() + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.send_header("Content-Disposition", 'attachment; filename="output.pdf"') + self.send_header("Content-Length", str(len(pdf_data))) + self.end_headers() + self.wfile.write(pdf_data) + except Exception as e: + self.send_error(500, str(e)) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + elif self.path == "/crawl": + url = data.get("url", "").strip() + if not url: + self._json(400, {"error": "No URL"}) + return + + import uuid + job = uuid.uuid4().hex[:12] + jdir = os.path.join(JOBS_DIR, job) + os.makedirs(jdir, exist_ok=True) + + # Write crawler script to npm dir so require() works + crawl_js = os.path.join(CWD, f"crawler_{job}.js") + with open(crawl_js, "w") as f: + f.write(CRAWLER_JS) + + # Write initial status + with open(os.path.join(jdir, "status.json"), "w") as f: + json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f) + + # Start background crawler + def run(): + subprocess.run( + ["timeout", "600", "node", crawl_js, url, jdir], + capture_output=True, text=True, cwd=CWD, + env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")} + ) + # Clean up script + try: os.unlink(crawl_js) + except: pass + + t = threading.Thread(target=run, daemon=True) + t.start() + + self._json(200, {"job": job, "status": "started"}) + + else: + self.send_error(404) + +HTTPServer(("127.0.0.1", 8099), Handler).serve_forever()