IT狗
e6116ecc7d
- crawl_hpe_v2.js: two-phase (discover versions + sub-pages: features/resolved/known/regulatory) - crawl_hpe.js: v1 single-phase - hpe-aos10-versions.json: 60 versions cache - server.py: - /url2pdf-tools/ now serves index.html (web UI) - /url2pdf-tools/<file> serves from /opt/url2pdf/ (with path traversal protection) - /url2pdf/crawl uses v2 crawler (was inline CRAWLER_JS hardcoded for Trend Micro) - index.html: new web UI (paste URL → crawl → download)
252 lines
9.6 KiB
JavaScript
252 lines
9.6 KiB
JavaScript
// Crawl HPE Aruba Networking AOS 10.x Release Notes - v2
|
|
// Phase 1: discover all version overview pages
|
|
// Phase 2: for each version, fetch features/resolved/known sub-pages
|
|
const puppeteer = require('puppeteer-core');
|
|
const fs = require('fs');
|
|
|
|
// Auto-detect Chrome across Mac / Linux / Windows
|
|
function findChrome() {
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const homedir = require('os').homedir();
|
|
const candidates = [
|
|
// macOS (local dev)
|
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.167/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
|
// Linux
|
|
'/usr/bin/google-chrome',
|
|
'/usr/bin/chromium-browser',
|
|
'/usr/bin/chromium',
|
|
// Docker puppeteer cache
|
|
path.join(homedir, '.cache/puppeteer/chrome/'),
|
|
];
|
|
for (const c of candidates) {
|
|
if (fs.existsSync(c)) return c;
|
|
}
|
|
return null;
|
|
}
|
|
const CHROME_PATH = findChrome();
|
|
if (!CHROME_PATH) {
|
|
console.error('No Chrome found. Try: apt install chromium / google-chrome-stable');
|
|
process.exit(1);
|
|
}
|
|
console.log('Using Chrome:', CHROME_PATH);
|
|
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
|
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
|
const VERSIONS_FILE = OUT_DIR + '/versions.json';
|
|
const MD_FILE = OUT_DIR + '/all-releases-v2.md';
|
|
|
|
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
|
|
|
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
|
const EXTRA_HEADERS = {
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br'
|
|
};
|
|
|
|
const SUB_PAGE_PATTERNS = [
|
|
{ regex: /features[/-]\d+\.htm/i, type: 'features' },
|
|
{ regex: /resolved-issues[/-]\d+\.htm/i, type: 'resolved' },
|
|
{ regex: /known-issues[/-]\d+\.htm/i, type: 'known' },
|
|
{ regex: /regulatory[/-]\d+\.htm/i, type: 'regulatory' },
|
|
];
|
|
|
|
async function setupPage(browser) {
|
|
const page = await browser.newPage();
|
|
await page.setUserAgent(USER_AGENT);
|
|
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
|
// Skip script/style in extraction
|
|
await page.evaluateOnNewDocument(() => {
|
|
window.MutationObserver = window.MutationObserver || undefined;
|
|
});
|
|
return page;
|
|
}
|
|
|
|
// Clean DOM extraction: skip script/style blocks
|
|
async function extractPage(page, href) {
|
|
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
|
await new Promise(r => setTimeout(r, 600));
|
|
|
|
return await page.evaluate(() => {
|
|
// Remove script + style + nav elements from clone
|
|
const document2 = document.cloneNode(true);
|
|
document2.querySelectorAll('script, style, nav, .MCWebHelpFramesetLink, .toolbar, .navigation').forEach(el => el.remove());
|
|
|
|
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.body-container'];
|
|
let main = document2.body;
|
|
for (const sel of mainSel) {
|
|
const el = document2.querySelector(sel);
|
|
if (el) { main = el; break; }
|
|
}
|
|
|
|
const h1 = main.querySelector('h1, h2');
|
|
const title = h1 ? h1.textContent.trim() : document2.title.trim();
|
|
|
|
const blocks = [];
|
|
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
|
const txt = el.textContent.trim();
|
|
if (!txt || txt.length < 3) return;
|
|
const tag = el.tagName;
|
|
if (tag.match(/^H[1-4]$/)) {
|
|
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
|
} else if (tag === 'LI') {
|
|
blocks.push('- ' + txt);
|
|
} else if (tag === 'PRE' || tag === 'CODE') {
|
|
blocks.push('\n```\n' + txt + '\n```\n');
|
|
} else if (tag === 'TABLE') {
|
|
const rows = [];
|
|
el.querySelectorAll('tr').forEach(tr => {
|
|
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
|
if (cells) rows.push('| ' + cells + ' |');
|
|
});
|
|
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
|
} else {
|
|
blocks.push(txt);
|
|
}
|
|
});
|
|
return { title, body: blocks.join('\n\n') };
|
|
});
|
|
}
|
|
|
|
async function discoverVersions(page) {
|
|
if (fs.existsSync(VERSIONS_FILE)) {
|
|
const versions = JSON.parse(fs.readFileSync(VERSIONS_FILE, 'utf8'));
|
|
console.log(`📋 ${versions.length} versions (cached)`);
|
|
return versions;
|
|
}
|
|
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
|
|
const baseUrl = new URL(BASE);
|
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
|
|
|
const allLinks = await page.evaluate(() => {
|
|
const links = [];
|
|
document.querySelectorAll('a[href]').forEach(a => {
|
|
const text = a.textContent.trim();
|
|
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
|
});
|
|
return links;
|
|
});
|
|
|
|
const seen = new Set();
|
|
const versions = [];
|
|
for (const link of allLinks) {
|
|
try {
|
|
const u = new URL(link.href);
|
|
if (u.hostname !== baseUrl.hostname) continue;
|
|
if (!u.pathname.startsWith(articleDir)) continue;
|
|
if (u.hash) continue;
|
|
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
|
const fn = u.pathname.split('/').pop();
|
|
if (!fn || fn.length < 3) continue;
|
|
if (!seen.has(fn)) {
|
|
seen.add(fn);
|
|
versions.push({ href: link.href, text: link.text });
|
|
}
|
|
} catch (e) {}
|
|
}
|
|
|
|
fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versions, null, 2));
|
|
console.log(`📋 ${versions.length} version pages found`);
|
|
return versions;
|
|
}
|
|
|
|
async function discoverSubPages(page, versionUrl) {
|
|
await page.goto(versionUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
|
await new Promise(r => setTimeout(r, 1500));
|
|
return await page.evaluate(() => {
|
|
const links = [];
|
|
document.querySelectorAll('a[href]').forEach(a => {
|
|
const href = a.href;
|
|
const text = a.textContent.trim();
|
|
if (/features|resolved-issues|known-issues|regulatory|whats-new/.test(href) && !/^#/.test(href) && !/Send Feedback/.test(text)) {
|
|
links.push({ href, text: text.slice(0, 80) });
|
|
}
|
|
});
|
|
return links;
|
|
});
|
|
}
|
|
|
|
async function crawl() {
|
|
const browser = await puppeteer.launch({
|
|
executablePath: CHROME_PATH,
|
|
headless: true,
|
|
args: CHROME_ARGS
|
|
});
|
|
const page = await setupPage(browser);
|
|
|
|
const versions = await discoverVersions(page);
|
|
console.log(`\n📥 Phase 1 (${versions.length} version overviews)...\n`);
|
|
|
|
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes (v2)\n\n> Source: ${BASE}\n> Versions: ${versions.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
|
|
|
let ok = 0, fail = 0;
|
|
const allSubPages = [];
|
|
|
|
for (let i = 0; i < versions.length; i++) {
|
|
const { href, text } = versions[i];
|
|
try {
|
|
const content = await extractPage(page, href);
|
|
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
|
fs.appendFileSync(MD_FILE, md);
|
|
ok++;
|
|
console.log(`✅ [${i+1}/${versions.length}] ${content.title.slice(0, 50).replace(/\n/g, ' ')}`);
|
|
|
|
// Phase 2: discover sub-pages for this version
|
|
const subPages = await discoverSubPages(page, href);
|
|
for (const sub of subPages) {
|
|
// Dedup: only add sub-pages for this specific version
|
|
if (sub.href.includes(href.split('/').slice(0, -1).join('/')) || sub.href.includes('/10.')) {
|
|
allSubPages.push({ parent: text, parentHref: href, ...sub });
|
|
}
|
|
}
|
|
} catch (e) {
|
|
fail++;
|
|
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
|
fs.appendFileSync(MD_FILE, md);
|
|
console.log(`❌ [${i+1}/${versions.length}] ${e.message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
|
|
// Dedupe sub-pages
|
|
const seenSub = new Set();
|
|
const uniqueSub = allSubPages.filter(s => {
|
|
if (seenSub.has(s.href)) return false;
|
|
seenSub.add(s.href);
|
|
return true;
|
|
});
|
|
|
|
console.log(`\n📥 Phase 2 (${uniqueSub.length} sub-pages: features/resolved/known)...\n`);
|
|
|
|
fs.appendFileSync(MD_FILE, `\n\n# === Detail Sub-Pages (Features / Resolved Issues / Known Issues) ===\n\n`);
|
|
|
|
let subOk = 0, subFail = 0;
|
|
for (let i = 0; i < uniqueSub.length; i++) {
|
|
const sub = uniqueSub[i];
|
|
const type = SUB_PAGE_PATTERNS.find(p => p.regex.test(sub.href))?.type || 'detail';
|
|
try {
|
|
const content = await extractPage(page, sub.href);
|
|
const md = `## ${sub.parent} — ${type.toUpperCase()}\n\n_${sub.text}_\n\n_Source: ${sub.href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
|
fs.appendFileSync(MD_FILE, md);
|
|
subOk++;
|
|
console.log(`✅ [${i+1}/${uniqueSub.length}] ${type.padEnd(10)} ${sub.parent} — ${sub.text.slice(0, 40)}`);
|
|
} catch (e) {
|
|
subFail++;
|
|
const md = `## ${sub.parent} — ${type}\n\n_❌ Error: ${e.message}_\n\n_Source: ${sub.href}_\n\n---\n\n`;
|
|
fs.appendFileSync(MD_FILE, md);
|
|
console.log(`❌ [${i+1}/${uniqueSub.length}] ${type} ${e.message.slice(0, 60)}`);
|
|
}
|
|
}
|
|
|
|
await browser.close();
|
|
console.log(`\n📊 Done:`);
|
|
console.log(` Phase 1 (overviews): ${ok} OK, ${fail} failed`);
|
|
console.log(` Phase 2 (sub-pages): ${subOk} OK, ${subFail} failed`);
|
|
console.log(`📁 Output: ${MD_FILE}`);
|
|
}
|
|
|
|
crawl().catch(err => { console.error(err); process.exit(1); });
|