v2: two-phase doc crawler + web UI + server.py static file serving
- crawl_hpe_v2.js: two-phase (discover versions + sub-pages: features/resolved/known/regulatory) - crawl_hpe.js: v1 single-phase - hpe-aos10-versions.json: 60 versions cache - server.py: - /url2pdf-tools/ now serves index.html (web UI) - /url2pdf-tools/<file> serves from /opt/url2pdf/ (with path traversal protection) - /url2pdf/crawl uses v2 crawler (was inline CRAWLER_JS hardcoded for Trend Micro) - index.html: new web UI (paste URL → crawl → download)
This commit is contained in:
IT狗
+158
@@ -0,0 +1,158 @@
|
|||||||
|
// Crawl HPE Aruba Networking AOS 10.x Release Notes
|
||||||
|
// Uses puppeteer-core + existing Chrome for Testing
|
||||||
|
const puppeteer = require('puppeteer-core');
|
||||||
|
const fs = require('fs');
|
||||||
|
|
||||||
|
const CHROME_PATH = '/Users/admin/.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing';
|
||||||
|
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
||||||
|
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
||||||
|
const URLS_FILE = OUT_DIR + '/urls.json';
|
||||||
|
const MD_FILE = OUT_DIR + '/all-releases.md';
|
||||||
|
|
||||||
|
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||||
|
|
||||||
|
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
||||||
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||||
|
const EXTRA_HEADERS = {
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
'Accept-Encoding': 'gzip, deflate, br'
|
||||||
|
};
|
||||||
|
|
||||||
|
async function setupPage(browser) {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.setUserAgent(USER_AGENT);
|
||||||
|
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAllUrls() {
|
||||||
|
if (fs.existsSync(URLS_FILE)) {
|
||||||
|
const urls = JSON.parse(fs.readFileSync(URLS_FILE, 'utf8'));
|
||||||
|
console.log(`📋 ${urls.length} URLs (cached)`);
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||||
|
await new Promise(r => setTimeout(r, 3000));
|
||||||
|
|
||||||
|
const baseUrl = new URL(BASE);
|
||||||
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
|
const allLinks = await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
|
||||||
|
const seen = new Set();
|
||||||
|
const docPages = [];
|
||||||
|
for (const link of allLinks) {
|
||||||
|
try {
|
||||||
|
const u = new URL(link.href);
|
||||||
|
if (u.hostname !== baseUrl.hostname) continue;
|
||||||
|
if (!u.pathname.startsWith(articleDir)) continue;
|
||||||
|
if (u.hash) continue;
|
||||||
|
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
||||||
|
const fn = u.pathname.split('/').pop();
|
||||||
|
if (!fn || fn.length < 3) continue;
|
||||||
|
if (!seen.has(fn)) {
|
||||||
|
seen.add(fn);
|
||||||
|
docPages.push({ href: link.href, text: link.text });
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
docPages.unshift({ href: BASE, text: 'All Releases (Index)' });
|
||||||
|
fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
|
||||||
|
console.log(`📋 ${docPages.length} URLs found`);
|
||||||
|
return docPages;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractPage(page, href) {
|
||||||
|
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 600));
|
||||||
|
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.MCWebHelpFramesetLink'];
|
||||||
|
let main = document.body;
|
||||||
|
for (const sel of mainSel) {
|
||||||
|
const el = document.querySelector(sel);
|
||||||
|
if (el) { main = el; break; }
|
||||||
|
}
|
||||||
|
const h1 = main.querySelector('h1, h2');
|
||||||
|
const title = h1 ? h1.textContent.trim() : document.title.trim();
|
||||||
|
|
||||||
|
const blocks = [];
|
||||||
|
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
||||||
|
const txt = el.textContent.trim();
|
||||||
|
if (!txt || txt.length < 3) return;
|
||||||
|
const tag = el.tagName;
|
||||||
|
if (tag.match(/^H[1-4]$/)) {
|
||||||
|
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
||||||
|
} else if (tag === 'LI') {
|
||||||
|
blocks.push('- ' + txt);
|
||||||
|
} else if (tag === 'PRE' || tag === 'CODE') {
|
||||||
|
blocks.push('\n```\n' + txt + '\n```\n');
|
||||||
|
} else if (tag === 'TABLE') {
|
||||||
|
const rows = [];
|
||||||
|
el.querySelectorAll('tr').forEach(tr => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
||||||
|
if (cells) rows.push('| ' + cells + ' |');
|
||||||
|
});
|
||||||
|
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
||||||
|
} else {
|
||||||
|
blocks.push(txt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return { title, body: blocks.join('\n\n') };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function crawl() {
|
||||||
|
const urls = await getAllUrls();
|
||||||
|
console.log(`\n📥 Starting crawl of ${urls.length} pages...\n`);
|
||||||
|
|
||||||
|
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes\n\n> Source: ${BASE}\n> Pages: ${urls.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
||||||
|
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
|
||||||
|
let ok = 0, fail = 0;
|
||||||
|
for (let i = 0; i < urls.length; i++) {
|
||||||
|
const { href, text } = urls[i];
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, href);
|
||||||
|
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
ok++;
|
||||||
|
console.log(`✅ [${i+1}/${urls.length}] ${content.title.slice(0, 60).replace(/\n/g, ' ')}`);
|
||||||
|
} catch (e) {
|
||||||
|
fail++;
|
||||||
|
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${urls.length}] ${e.message.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
console.log(`\n📊 Done: ${ok} OK, ${fail} failed`);
|
||||||
|
console.log(`📁 Output: ${MD_FILE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
crawl().catch(err => { console.error(err); process.exit(1); });
|
||||||
+251
@@ -0,0 +1,251 @@
|
|||||||
|
// Crawl HPE Aruba Networking AOS 10.x Release Notes - v2
|
||||||
|
// Phase 1: discover all version overview pages
|
||||||
|
// Phase 2: for each version, fetch features/resolved/known sub-pages
|
||||||
|
const puppeteer = require('puppeteer-core');
|
||||||
|
const fs = require('fs');
|
||||||
|
|
||||||
|
// Auto-detect Chrome across Mac / Linux / Windows
|
||||||
|
function findChrome() {
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const homedir = require('os').homedir();
|
||||||
|
const candidates = [
|
||||||
|
// macOS (local dev)
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.167/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
||||||
|
// Linux
|
||||||
|
'/usr/bin/google-chrome',
|
||||||
|
'/usr/bin/chromium-browser',
|
||||||
|
'/usr/bin/chromium',
|
||||||
|
// Docker puppeteer cache
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/'),
|
||||||
|
];
|
||||||
|
for (const c of candidates) {
|
||||||
|
if (fs.existsSync(c)) return c;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const CHROME_PATH = findChrome();
|
||||||
|
if (!CHROME_PATH) {
|
||||||
|
console.error('No Chrome found. Try: apt install chromium / google-chrome-stable');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log('Using Chrome:', CHROME_PATH);
|
||||||
|
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
||||||
|
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
||||||
|
const VERSIONS_FILE = OUT_DIR + '/versions.json';
|
||||||
|
const MD_FILE = OUT_DIR + '/all-releases-v2.md';
|
||||||
|
|
||||||
|
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||||
|
|
||||||
|
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
||||||
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||||
|
const EXTRA_HEADERS = {
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
'Accept-Encoding': 'gzip, deflate, br'
|
||||||
|
};
|
||||||
|
|
||||||
|
const SUB_PAGE_PATTERNS = [
|
||||||
|
{ regex: /features[/-]\d+\.htm/i, type: 'features' },
|
||||||
|
{ regex: /resolved-issues[/-]\d+\.htm/i, type: 'resolved' },
|
||||||
|
{ regex: /known-issues[/-]\d+\.htm/i, type: 'known' },
|
||||||
|
{ regex: /regulatory[/-]\d+\.htm/i, type: 'regulatory' },
|
||||||
|
];
|
||||||
|
|
||||||
|
async function setupPage(browser) {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.setUserAgent(USER_AGENT);
|
||||||
|
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
||||||
|
// Skip script/style in extraction
|
||||||
|
await page.evaluateOnNewDocument(() => {
|
||||||
|
window.MutationObserver = window.MutationObserver || undefined;
|
||||||
|
});
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean DOM extraction: skip script/style blocks
|
||||||
|
async function extractPage(page, href) {
|
||||||
|
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 600));
|
||||||
|
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
// Remove script + style + nav elements from clone
|
||||||
|
const document2 = document.cloneNode(true);
|
||||||
|
document2.querySelectorAll('script, style, nav, .MCWebHelpFramesetLink, .toolbar, .navigation').forEach(el => el.remove());
|
||||||
|
|
||||||
|
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.body-container'];
|
||||||
|
let main = document2.body;
|
||||||
|
for (const sel of mainSel) {
|
||||||
|
const el = document2.querySelector(sel);
|
||||||
|
if (el) { main = el; break; }
|
||||||
|
}
|
||||||
|
|
||||||
|
const h1 = main.querySelector('h1, h2');
|
||||||
|
const title = h1 ? h1.textContent.trim() : document2.title.trim();
|
||||||
|
|
||||||
|
const blocks = [];
|
||||||
|
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
||||||
|
const txt = el.textContent.trim();
|
||||||
|
if (!txt || txt.length < 3) return;
|
||||||
|
const tag = el.tagName;
|
||||||
|
if (tag.match(/^H[1-4]$/)) {
|
||||||
|
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
||||||
|
} else if (tag === 'LI') {
|
||||||
|
blocks.push('- ' + txt);
|
||||||
|
} else if (tag === 'PRE' || tag === 'CODE') {
|
||||||
|
blocks.push('\n```\n' + txt + '\n```\n');
|
||||||
|
} else if (tag === 'TABLE') {
|
||||||
|
const rows = [];
|
||||||
|
el.querySelectorAll('tr').forEach(tr => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
||||||
|
if (cells) rows.push('| ' + cells + ' |');
|
||||||
|
});
|
||||||
|
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
||||||
|
} else {
|
||||||
|
blocks.push(txt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return { title, body: blocks.join('\n\n') };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverVersions(page) {
|
||||||
|
if (fs.existsSync(VERSIONS_FILE)) {
|
||||||
|
const versions = JSON.parse(fs.readFileSync(VERSIONS_FILE, 'utf8'));
|
||||||
|
console.log(`📋 ${versions.length} versions (cached)`);
|
||||||
|
return versions;
|
||||||
|
}
|
||||||
|
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||||
|
await new Promise(r => setTimeout(r, 3000));
|
||||||
|
|
||||||
|
const baseUrl = new URL(BASE);
|
||||||
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
|
const allLinks = await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
|
||||||
|
const seen = new Set();
|
||||||
|
const versions = [];
|
||||||
|
for (const link of allLinks) {
|
||||||
|
try {
|
||||||
|
const u = new URL(link.href);
|
||||||
|
if (u.hostname !== baseUrl.hostname) continue;
|
||||||
|
if (!u.pathname.startsWith(articleDir)) continue;
|
||||||
|
if (u.hash) continue;
|
||||||
|
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
||||||
|
const fn = u.pathname.split('/').pop();
|
||||||
|
if (!fn || fn.length < 3) continue;
|
||||||
|
if (!seen.has(fn)) {
|
||||||
|
seen.add(fn);
|
||||||
|
versions.push({ href: link.href, text: link.text });
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versions, null, 2));
|
||||||
|
console.log(`📋 ${versions.length} version pages found`);
|
||||||
|
return versions;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverSubPages(page, versionUrl) {
|
||||||
|
await page.goto(versionUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 1500));
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const href = a.href;
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (/features|resolved-issues|known-issues|regulatory|whats-new/.test(href) && !/^#/.test(href) && !/Send Feedback/.test(text)) {
|
||||||
|
links.push({ href, text: text.slice(0, 80) });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function crawl() {
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
|
||||||
|
const versions = await discoverVersions(page);
|
||||||
|
console.log(`\n📥 Phase 1 (${versions.length} version overviews)...\n`);
|
||||||
|
|
||||||
|
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes (v2)\n\n> Source: ${BASE}\n> Versions: ${versions.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
||||||
|
|
||||||
|
let ok = 0, fail = 0;
|
||||||
|
const allSubPages = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < versions.length; i++) {
|
||||||
|
const { href, text } = versions[i];
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, href);
|
||||||
|
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
ok++;
|
||||||
|
console.log(`✅ [${i+1}/${versions.length}] ${content.title.slice(0, 50).replace(/\n/g, ' ')}`);
|
||||||
|
|
||||||
|
// Phase 2: discover sub-pages for this version
|
||||||
|
const subPages = await discoverSubPages(page, href);
|
||||||
|
for (const sub of subPages) {
|
||||||
|
// Dedup: only add sub-pages for this specific version
|
||||||
|
if (sub.href.includes(href.split('/').slice(0, -1).join('/')) || sub.href.includes('/10.')) {
|
||||||
|
allSubPages.push({ parent: text, parentHref: href, ...sub });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
fail++;
|
||||||
|
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${versions.length}] ${e.message.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dedupe sub-pages
|
||||||
|
const seenSub = new Set();
|
||||||
|
const uniqueSub = allSubPages.filter(s => {
|
||||||
|
if (seenSub.has(s.href)) return false;
|
||||||
|
seenSub.add(s.href);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`\n📥 Phase 2 (${uniqueSub.length} sub-pages: features/resolved/known)...\n`);
|
||||||
|
|
||||||
|
fs.appendFileSync(MD_FILE, `\n\n# === Detail Sub-Pages (Features / Resolved Issues / Known Issues) ===\n\n`);
|
||||||
|
|
||||||
|
let subOk = 0, subFail = 0;
|
||||||
|
for (let i = 0; i < uniqueSub.length; i++) {
|
||||||
|
const sub = uniqueSub[i];
|
||||||
|
const type = SUB_PAGE_PATTERNS.find(p => p.regex.test(sub.href))?.type || 'detail';
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, sub.href);
|
||||||
|
const md = `## ${sub.parent} — ${type.toUpperCase()}\n\n_${sub.text}_\n\n_Source: ${sub.href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
subOk++;
|
||||||
|
console.log(`✅ [${i+1}/${uniqueSub.length}] ${type.padEnd(10)} ${sub.parent} — ${sub.text.slice(0, 40)}`);
|
||||||
|
} catch (e) {
|
||||||
|
subFail++;
|
||||||
|
const md = `## ${sub.parent} — ${type}\n\n_❌ Error: ${e.message}_\n\n_Source: ${sub.href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${uniqueSub.length}] ${type} ${e.message.slice(0, 60)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
console.log(`\n📊 Done:`);
|
||||||
|
console.log(` Phase 1 (overviews): ${ok} OK, ${fail} failed`);
|
||||||
|
console.log(` Phase 2 (sub-pages): ${subOk} OK, ${subFail} failed`);
|
||||||
|
console.log(`📁 Output: ${MD_FILE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
crawl().catch(err => { console.error(err); process.exit(1); });
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/Home.htm",
|
||||||
|
"text": "Home"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/common%20files/topic_files/whats_new.htm",
|
||||||
|
"text": "What's New"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm",
|
||||||
|
"text": "All Releases"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/common%20files/topic_files/terminology-change.htm",
|
||||||
|
"text": "Terminology Change"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/overview-1031x.htm",
|
||||||
|
"text": "10.3.1.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/overview-1040x.htm",
|
||||||
|
"text": "10.4.0.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/overview-1041x.htm",
|
||||||
|
"text": "10.4.1.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/overview-1050x.htm",
|
||||||
|
"text": "10.5.0.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/overview-1051x.htm",
|
||||||
|
"text": "10.5.1.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/overview-1060x.htm",
|
||||||
|
"text": "10.6.0.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/overview-1070x.htm",
|
||||||
|
"text": "10.7.0.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/overview-1071x.htm",
|
||||||
|
"text": "10.7.1.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/overview-1072x.htm",
|
||||||
|
"text": "10.7.2.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/overview-108x.htm",
|
||||||
|
"text": "10.8.x.x"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/10/overview-10810.htm",
|
||||||
|
"text": "10.8.1.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/02/overview-10802.htm",
|
||||||
|
"text": "10.8.0.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/01/overview-10801.htm",
|
||||||
|
"text": "10.8.0.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/00/overview-10800.htm",
|
||||||
|
"text": "10.8.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/6/overview-10726.htm",
|
||||||
|
"text": "10.7.2.6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/5/overview-10725.htm",
|
||||||
|
"text": "10.7.2.5"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/4/overview-10724.htm",
|
||||||
|
"text": "10.7.2.4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/3/overview-10723.htm",
|
||||||
|
"text": "10.7.2.3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/2/overview-10722.htm",
|
||||||
|
"text": "10.7.2.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/1/overview-10721.htm",
|
||||||
|
"text": "10.7.2.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/0/overview-10720.htm",
|
||||||
|
"text": "10.7.2.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/1/overview-10711.htm",
|
||||||
|
"text": "10.7.1.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/0/overview-10710.htm",
|
||||||
|
"text": "10.7.1.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/02/overview-10702.htm",
|
||||||
|
"text": "10.7.0.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/01/overview-10701.htm",
|
||||||
|
"text": "10.7.0.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/00/overview-10700.htm",
|
||||||
|
"text": "10.7.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/03/overview-10603.htm",
|
||||||
|
"text": "10.6.0.3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/02/overview-10602.htm",
|
||||||
|
"text": "10.6.0.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/01/overview-10601.htm",
|
||||||
|
"text": "10.6.0.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/00/overview-10600.htm",
|
||||||
|
"text": "10.6.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/1/overview-10511.htm",
|
||||||
|
"text": "10.5.1.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/0/overview-10510.htm",
|
||||||
|
"text": "10.5.1.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/01/overview-10501.htm",
|
||||||
|
"text": "10.5.0.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/00/overview-10500.htm",
|
||||||
|
"text": "10.5.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/12/overview-104112.htm",
|
||||||
|
"text": "10.4.1.12"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/11/overview-104111.htm",
|
||||||
|
"text": "10.4.1.11"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/10/overview-104110.htm",
|
||||||
|
"text": "10.4.1.10"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/9/overview-10419.htm",
|
||||||
|
"text": "10.4.1.9"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/8/overview-10418.htm",
|
||||||
|
"text": "10.4.1.8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/7/overview-10417.htm",
|
||||||
|
"text": "10.4.1.7"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/6/overview-10416.htm",
|
||||||
|
"text": "10.4.1.6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/5/overview-10415.htm",
|
||||||
|
"text": "10.4.1.5"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/4/overview-10414.htm",
|
||||||
|
"text": "10.4.1.4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/3/overview-10413.htm",
|
||||||
|
"text": "10.4.1.3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/2/overview-10412.htm",
|
||||||
|
"text": "10.4.1.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/1/overview-10411.htm",
|
||||||
|
"text": "10.4.1.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/0/overview-10410.htm",
|
||||||
|
"text": "10.4.1.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/03/overview-10403.htm",
|
||||||
|
"text": "10.4.0.3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/02/overview-10402.htm",
|
||||||
|
"text": "10.4.0.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/01/overview-10401.htm",
|
||||||
|
"text": "10.4.0.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/00/overview-10400.htm",
|
||||||
|
"text": "10.4.0.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/4/overview-10314.htm",
|
||||||
|
"text": "10.3.1.4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/3/overview-10313.htm",
|
||||||
|
"text": "10.3.1.3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/2/overview-10312.htm",
|
||||||
|
"text": "10.3.1.2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/1/overview-10311.htm",
|
||||||
|
"text": "10.3.1.1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/0/overview-10310.htm",
|
||||||
|
"text": "10.3.1.0"
|
||||||
|
}
|
||||||
|
]
|
||||||
+142
@@ -0,0 +1,142 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>URL-to-PDF · Doc Crawler</title>
|
||||||
|
<style>
|
||||||
|
*{box-sizing:border-box;margin:0;padding:0}
|
||||||
|
body{font-family:-apple-system,system-ui,sans-serif;background:#0a0a0f;color:#f0f0f4;min-height:100vh;display:flex;align-items:center;justify-content:center;padding:20px}
|
||||||
|
.card{background:#131317;border:1px solid #2a2a33;border-radius:16px;padding:32px;max-width:720px;width:100%}
|
||||||
|
h1{font-size:1.5rem;font-weight:700;margin-bottom:6px;letter-spacing:-0.3px}
|
||||||
|
.sub{color:#88889a;font-size:0.85rem;margin-bottom:24px}
|
||||||
|
.tag{display:inline-block;padding:3px 10px;border-radius:6px;background:rgba(139,92,246,0.15);color:#a78bfa;font-size:0.7rem;font-weight:600;margin-right:6px;margin-bottom:8px}
|
||||||
|
.row{display:flex;gap:8px;margin-bottom:12px}
|
||||||
|
input{flex:1;padding:12px 16px;border-radius:10px;border:1px solid #2a2a33;background:#0a0a0f;color:#f0f0f4;font-size:0.9rem;font-family:inherit;transition:.2s}
|
||||||
|
input:focus{outline:none;border-color:#8b5cf6;box-shadow:0 0 0 3px rgba(139,92,246,0.1)}
|
||||||
|
button{padding:12px 20px;border-radius:10px;border:none;background:#8b5cf6;color:#fff;font-size:0.9rem;font-weight:600;cursor:pointer;font-family:inherit;transition:.2s;white-space:nowrap}
|
||||||
|
button:hover{background:#7c3aed}
|
||||||
|
button:disabled{opacity:0.4;cursor:not-allowed}
|
||||||
|
.examples{background:#0a0a0f;border:1px solid #2a2a33;border-radius:10px;padding:12px;margin-bottom:16px;font-size:0.8rem}
|
||||||
|
.examples h3{font-size:0.75rem;color:#88889a;margin-bottom:8px;text-transform:uppercase;letter-spacing:0.5px}
|
||||||
|
.examples a{color:#a78bfa;text-decoration:none;display:block;padding:4px 0;font-family:monospace;font-size:0.75rem}
|
||||||
|
.examples a:hover{color:#8b5cf6}
|
||||||
|
.status{margin-top:16px;padding:14px 16px;border-radius:10px;font-size:0.85rem;display:none}
|
||||||
|
.status.active{display:block}
|
||||||
|
.status.info{background:rgba(96,165,250,0.1);color:#60a5fa;border:1px solid rgba(96,165,250,0.3)}
|
||||||
|
.status.success{background:rgba(16,185,129,0.1);color:#34d399;border:1px solid rgba(16,185,129,0.3)}
|
||||||
|
.status.error{background:rgba(239,68,68,0.1);color:#f87171;border:1px solid rgba(239,68,68,0.3)}
|
||||||
|
.progress{height:4px;background:#1a1a22;border-radius:2px;margin-top:8px;overflow:hidden}
|
||||||
|
.progress-bar{height:100%;background:#8b5cf6;border-radius:2px;transition:width .3s;width:0}
|
||||||
|
.result{margin-top:12px;font-size:0.8rem}
|
||||||
|
.result a{color:#34d399;text-decoration:none;font-weight:600}
|
||||||
|
.result a:hover{text-decoration:underline}
|
||||||
|
.info-box{background:#0a0a0f;border:1px solid #2a2a33;border-radius:10px;padding:14px;margin-bottom:16px;font-size:0.78rem;color:#bcbcc8;line-height:1.6}
|
||||||
|
.info-box code{background:#1a1a22;padding:2px 6px;border-radius:4px;font-size:0.72rem;color:#a78bfa}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="card">
|
||||||
|
<h1>📄 URL-to-PDF · Doc Crawler</h1>
|
||||||
|
<p class="sub">Paste a documentation URL → get all sub-pages as Markdown</p>
|
||||||
|
|
||||||
|
<div class="info-box">
|
||||||
|
🐕 <strong>IT狗 2026-07-24</strong> — Two-phase crawler: discovers version pages, then fetches each version's <em>features / resolved / known issues / regulatory</em> sub-pages. Currently tested with HPE AOS 10.x (209 pages).
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<span class="tag">v2</span><span class="tag">Two-phase</span><span class="tag">Markdown</span>
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<input id="urlInput" type="url" placeholder="https://example.com/docs/all-releases.htm" value="https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm">
|
||||||
|
<button id="goBtn" onclick="startCrawl()">Crawl</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="examples">
|
||||||
|
<h3>📚 Tested / Compatible</h3>
|
||||||
|
<a href="#" onclick="event.preventDefault();document.getElementById(\"urlInput\").value=this.textContent">https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm</a>
|
||||||
|
<a href="#" onclick="event.preventDefault();document.getElementById(\"urlInput\").value=this.textContent">https://arubanetworking.hpe.com/techdocs/AOS_8.x_RN_WebHelp/Content/all-releases.htm</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="status" class="status"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const API = "/url2pdf";
|
||||||
|
const input = document.getElementById("urlInput");
|
||||||
|
const btn = document.getElementById("goBtn");
|
||||||
|
const status = document.getElementById("status");
|
||||||
|
|
||||||
|
async function startCrawl() {
|
||||||
|
const url = (input.value || "").trim();
|
||||||
|
if (!url) { showStatus("error", "Please enter a URL"); return; }
|
||||||
|
if (!/^https?:\/\//.test(url)) { showStatus("error", "URL must start with http:// or https://"); return; }
|
||||||
|
|
||||||
|
btn.disabled = true;
|
||||||
|
showStatus("info", "🚀 Starting crawl...");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const r = await fetch(API + "/crawl", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {"Content-Type": "application/json"},
|
||||||
|
body: JSON.stringify({url})
|
||||||
|
});
|
||||||
|
const data = await r.json();
|
||||||
|
if (data.error) throw new Error(data.error);
|
||||||
|
|
||||||
|
const job = data.job;
|
||||||
|
showStatus("info", `⏳ Crawl started (job: <code>${job}</code>). Polling status...`);
|
||||||
|
|
||||||
|
// Poll status
|
||||||
|
const startTime = Date.now();
|
||||||
|
let pollCount = 0;
|
||||||
|
while (true) {
|
||||||
|
await new Promise(r => setTimeout(r, 5000));
|
||||||
|
pollCount++;
|
||||||
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||||
|
|
||||||
|
const sr = await fetch(API + "/crawl/status?job=" + job);
|
||||||
|
const sd = await sr.json();
|
||||||
|
|
||||||
|
const statusText = sd.status || "pending";
|
||||||
|
const sizeMB = sd.output_size ? (sd.output_size / 1024 / 1024).toFixed(1) + " MB" : "";
|
||||||
|
|
||||||
|
if (statusText === "done") {
|
||||||
|
showStatus("success", `✅ Done in ${elapsed}s — ${sizeMB} of markdown ready`, job);
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
} else if (statusText.startsWith("failed")) {
|
||||||
|
showStatus("error", `❌ Failed: ${statusText}<br><pre style="white-space:pre-wrap;font-size:0.7rem;color:#f87171;margin-top:8px;max-height:200px;overflow:auto">${sd.log_tail || ""}</pre>`);
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
} else if (statusText.startsWith("done:")) {
|
||||||
|
// legacy format
|
||||||
|
showStatus("success", `✅ ${statusText} (${elapsed}s)`, job);
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
showStatus("info", `⏳ ${statusText} (${elapsed}s elapsed, poll #${pollCount})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (elapsed > 1200) { // 20min timeout
|
||||||
|
showStatus("error", "⏱️ Timeout (>20min). Check job manually.");
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
showStatus("error", "❌ Error: " + e.message);
|
||||||
|
btn.disabled = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showStatus(type, html, job) {
|
||||||
|
status.className = "status active " + type;
|
||||||
|
const jobLink = job ? `<div class="result">📥 <a href="${API}/crawl/download?job=${job}&format=md" download>Download output.md</a></div>` : "";
|
||||||
|
status.innerHTML = html + jobLink;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allow Enter key
|
||||||
|
input.addEventListener("keypress", e => { if (e.key === "Enter") startCrawl(); });
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -313,6 +313,43 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
self.wfile.write(json.dumps(data).encode())
|
self.wfile.write(json.dumps(data).encode())
|
||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
|
# Static file serving: <file> or /url2pdf-tools/<file> → /opt/url2pdf/<file>
|
||||||
|
# (nginx strips /url2pdf prefix; we accept both with and without it)
|
||||||
|
# Static file serving: <file> or /url2pdf-tools/<file> -> /opt/url2pdf/<file>
|
||||||
|
# (nginx strips /url2pdf prefix; we accept both with and without it)
|
||||||
|
if self.path.startswith("/url2pdf-tools/"):
|
||||||
|
rel = self.path[len("/url2pdf-tools/"):]
|
||||||
|
elif "." in self.path.split("?")[0].split("/")[-1]:
|
||||||
|
rel = self.path.lstrip("/")
|
||||||
|
else:
|
||||||
|
rel = None
|
||||||
|
# Always serve index.html for root paths or empty
|
||||||
|
if rel is None or rel == "" or rel == "/":
|
||||||
|
rel = "index.html"
|
||||||
|
# Prevent path traversal
|
||||||
|
safe = os.path.normpath(rel).lstrip("/")
|
||||||
|
if safe.startswith("..") or "/.." in safe:
|
||||||
|
self.send_error(400, "Bad path")
|
||||||
|
return
|
||||||
|
fp = os.path.join("/opt/url2pdf", safe)
|
||||||
|
fp = os.path.join("/opt/url2pdf", safe)
|
||||||
|
if os.path.isfile(fp):
|
||||||
|
import mimetypes
|
||||||
|
ct, _ = mimetypes.guess_type(fp)
|
||||||
|
if ct is None:
|
||||||
|
ct = "application/octet-stream"
|
||||||
|
with open(fp, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", ct)
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
return
|
||||||
|
# file not found under /opt/url2pdf - fall through to API or HTML fallback
|
||||||
|
self.send_error(404, "Not found: " + fp)
|
||||||
|
return
|
||||||
|
|
||||||
if self.path.startswith("/crawl/status"):
|
if self.path.startswith("/crawl/status"):
|
||||||
qs = urllib.parse.urlparse(self.path).query
|
qs = urllib.parse.urlparse(self.path).query
|
||||||
params = urllib.parse.parse_qs(qs)
|
params = urllib.parse.parse_qs(qs)
|
||||||
@@ -327,6 +364,7 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
else:
|
else:
|
||||||
self._json(200, {"status": "pending"})
|
self._json(200, {"status": "pending"})
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
if self.path.startswith("/crawl/download"):
|
if self.path.startswith("/crawl/download"):
|
||||||
qs = urllib.parse.urlparse(self.path).query
|
qs = urllib.parse.urlparse(self.path).query
|
||||||
@@ -424,30 +462,42 @@ class Handler(BaseHTTPRequestHandler):
|
|||||||
jdir = os.path.join(JOBS_DIR, job)
|
jdir = os.path.join(JOBS_DIR, job)
|
||||||
os.makedirs(jdir, exist_ok=True)
|
os.makedirs(jdir, exist_ok=True)
|
||||||
|
|
||||||
# Write crawler script to npm dir so require() works
|
# Use v2 crawler (two-phase: discover versions + sub-pages)
|
||||||
crawl_js = os.path.join(CWD, f"crawler_{job}.js")
|
v2_crawler = os.path.join(CWD, "crawl_hpe_v2.js")
|
||||||
with open(crawl_js, "w") as f:
|
if not os.path.exists(v2_crawler):
|
||||||
f.write(CRAWLER_JS)
|
self._json(500, {"error": "v2 crawler not installed at /opt/url2pdf/crawl_hpe_v2.js"})
|
||||||
|
return
|
||||||
|
|
||||||
# Write initial status
|
# Write initial status
|
||||||
with open(os.path.join(jdir, "status.json"), "w") as f:
|
with open(os.path.join(jdir, "status.json"), "w") as f:
|
||||||
json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
|
json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
|
||||||
|
|
||||||
# Start background crawler
|
# Start background crawler (v2 writes output.md to /tmp/hpe-aruba-aos10-rn/, so copy to jdir)
|
||||||
def run():
|
def run():
|
||||||
subprocess.run(
|
r = subprocess.run(
|
||||||
["timeout", "600", "node", crawl_js, url, jdir],
|
["timeout", "900", "node", v2_crawler, url],
|
||||||
capture_output=True, text=True, cwd=CWD,
|
capture_output=True, text=True, cwd=CWD,
|
||||||
env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
|
env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
|
||||||
)
|
)
|
||||||
# Clean up script
|
# Copy v2 output to jdir
|
||||||
try: os.unlink(crawl_js)
|
import shutil
|
||||||
except: pass
|
src_md = "/tmp/hpe-aruba-aos10-rn/all-releases-v2.md"
|
||||||
|
dst_md = os.path.join(jdir, "output.md")
|
||||||
|
if os.path.exists(src_md):
|
||||||
|
shutil.copy(src_md, dst_md)
|
||||||
|
# Update status
|
||||||
|
with open(os.path.join(jdir, "status.json"), "w") as f:
|
||||||
|
json.dump({
|
||||||
|
"status": "done" if r.returncode == 0 else f"failed:{r.returncode}",
|
||||||
|
"time": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
|
"output_size": os.path.getsize(dst_md) if os.path.exists(dst_md) else 0,
|
||||||
|
"log_tail": r.stdout[-500:] if r.stdout else r.stderr[-500:]
|
||||||
|
}, f)
|
||||||
|
|
||||||
t = threading.Thread(target=run, daemon=True)
|
t = threading.Thread(target=run, daemon=True)
|
||||||
t.start()
|
t.start()
|
||||||
|
|
||||||
self._json(200, {"job": job, "status": "started"})
|
self._json(200, {"job": job, "status": "started", "crawler": "v2"})
|
||||||
|
|
||||||
else:
|
else:
|
||||||
self.send_error(404)
|
self.send_error(404)
|
||||||
|
|||||||
Reference in New Issue
Block a user