v2: two-phase doc crawler + web UI + server.py static file serving
- crawl_hpe_v2.js: two-phase (discover versions + sub-pages: features/resolved/known/regulatory) - crawl_hpe.js: v1 single-phase - hpe-aos10-versions.json: 60 versions cache - server.py: - /url2pdf-tools/ now serves index.html (web UI) - /url2pdf-tools/<file> serves from /opt/url2pdf/ (with path traversal protection) - /url2pdf/crawl uses v2 crawler (was inline CRAWLER_JS hardcoded for Trend Micro) - index.html: new web UI (paste URL → crawl → download)
This commit is contained in:
IT狗
+158
@@ -0,0 +1,158 @@
|
||||
// Crawl HPE Aruba Networking AOS 10.x Release Notes
|
||||
// Uses puppeteer-core + existing Chrome for Testing
|
||||
const puppeteer = require('puppeteer-core');
|
||||
const fs = require('fs');
|
||||
|
||||
const CHROME_PATH = '/Users/admin/.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing';
|
||||
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
||||
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
||||
const URLS_FILE = OUT_DIR + '/urls.json';
|
||||
const MD_FILE = OUT_DIR + '/all-releases.md';
|
||||
|
||||
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
||||
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||
const EXTRA_HEADERS = {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br'
|
||||
};
|
||||
|
||||
async function setupPage(browser) {
|
||||
const page = await browser.newPage();
|
||||
await page.setUserAgent(USER_AGENT);
|
||||
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
||||
return page;
|
||||
}
|
||||
|
||||
async function getAllUrls() {
|
||||
if (fs.existsSync(URLS_FILE)) {
|
||||
const urls = JSON.parse(fs.readFileSync(URLS_FILE, 'utf8'));
|
||||
console.log(`📋 ${urls.length} URLs (cached)`);
|
||||
return urls;
|
||||
}
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: CHROME_PATH,
|
||||
headless: true,
|
||||
args: CHROME_ARGS
|
||||
});
|
||||
const page = await setupPage(browser);
|
||||
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await new Promise(r => setTimeout(r, 3000));
|
||||
|
||||
const baseUrl = new URL(BASE);
|
||||
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
||||
|
||||
const allLinks = await page.evaluate(() => {
|
||||
const links = [];
|
||||
document.querySelectorAll('a[href]').forEach(a => {
|
||||
const text = a.textContent.trim();
|
||||
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
||||
});
|
||||
return links;
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
|
||||
const seen = new Set();
|
||||
const docPages = [];
|
||||
for (const link of allLinks) {
|
||||
try {
|
||||
const u = new URL(link.href);
|
||||
if (u.hostname !== baseUrl.hostname) continue;
|
||||
if (!u.pathname.startsWith(articleDir)) continue;
|
||||
if (u.hash) continue;
|
||||
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
||||
const fn = u.pathname.split('/').pop();
|
||||
if (!fn || fn.length < 3) continue;
|
||||
if (!seen.has(fn)) {
|
||||
seen.add(fn);
|
||||
docPages.push({ href: link.href, text: link.text });
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
docPages.unshift({ href: BASE, text: 'All Releases (Index)' });
|
||||
fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
|
||||
console.log(`📋 ${docPages.length} URLs found`);
|
||||
return docPages;
|
||||
}
|
||||
|
||||
async function extractPage(page, href) {
|
||||
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 600));
|
||||
|
||||
return await page.evaluate(() => {
|
||||
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.MCWebHelpFramesetLink'];
|
||||
let main = document.body;
|
||||
for (const sel of mainSel) {
|
||||
const el = document.querySelector(sel);
|
||||
if (el) { main = el; break; }
|
||||
}
|
||||
const h1 = main.querySelector('h1, h2');
|
||||
const title = h1 ? h1.textContent.trim() : document.title.trim();
|
||||
|
||||
const blocks = [];
|
||||
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
||||
const txt = el.textContent.trim();
|
||||
if (!txt || txt.length < 3) return;
|
||||
const tag = el.tagName;
|
||||
if (tag.match(/^H[1-4]$/)) {
|
||||
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
||||
} else if (tag === 'LI') {
|
||||
blocks.push('- ' + txt);
|
||||
} else if (tag === 'PRE' || tag === 'CODE') {
|
||||
blocks.push('\n```\n' + txt + '\n```\n');
|
||||
} else if (tag === 'TABLE') {
|
||||
const rows = [];
|
||||
el.querySelectorAll('tr').forEach(tr => {
|
||||
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
||||
if (cells) rows.push('| ' + cells + ' |');
|
||||
});
|
||||
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
||||
} else {
|
||||
blocks.push(txt);
|
||||
}
|
||||
});
|
||||
return { title, body: blocks.join('\n\n') };
|
||||
});
|
||||
}
|
||||
|
||||
async function crawl() {
|
||||
const urls = await getAllUrls();
|
||||
console.log(`\n📥 Starting crawl of ${urls.length} pages...\n`);
|
||||
|
||||
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes\n\n> Source: ${BASE}\n> Pages: ${urls.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: CHROME_PATH,
|
||||
headless: true,
|
||||
args: CHROME_ARGS
|
||||
});
|
||||
const page = await setupPage(browser);
|
||||
|
||||
let ok = 0, fail = 0;
|
||||
for (let i = 0; i < urls.length; i++) {
|
||||
const { href, text } = urls[i];
|
||||
try {
|
||||
const content = await extractPage(page, href);
|
||||
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||
fs.appendFileSync(MD_FILE, md);
|
||||
ok++;
|
||||
console.log(`✅ [${i+1}/${urls.length}] ${content.title.slice(0, 60).replace(/\n/g, ' ')}`);
|
||||
} catch (e) {
|
||||
fail++;
|
||||
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
||||
fs.appendFileSync(MD_FILE, md);
|
||||
console.log(`❌ [${i+1}/${urls.length}] ${e.message.slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
console.log(`\n📊 Done: ${ok} OK, ${fail} failed`);
|
||||
console.log(`📁 Output: ${MD_FILE}`);
|
||||
}
|
||||
|
||||
crawl().catch(err => { console.error(err); process.exit(1); });
|
||||
Reference in New Issue
Block a user