IT狗 2026-07-24: HPE AOS 10.x doc crawler (v1 + v2)
- crawl_hpe.js: v1 single-phase (Phase 1 only) - crawl_hpe_v2.js: v2 two-phase (discover versions + sub-pages) - v2: auto-detects Chrome across Mac/Linux, sets fake UA, handles cloudflare-like Access Denied - covered 209 pages for HPE AOS 10.x (60 versions + 149 sub-pages: features/resolved/known/regulatory) - cumulative: 2.5MB markdown output, all sub-page sections parsed
This commit is contained in:
IT狗
+158
@@ -0,0 +1,158 @@
|
|||||||
|
// Crawl HPE Aruba Networking AOS 10.x Release Notes
|
||||||
|
// Uses puppeteer-core + existing Chrome for Testing
|
||||||
|
const puppeteer = require('puppeteer-core');
|
||||||
|
const fs = require('fs');
|
||||||
|
|
||||||
|
const CHROME_PATH = '/Users/admin/.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing';
|
||||||
|
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
||||||
|
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
||||||
|
const URLS_FILE = OUT_DIR + '/urls.json';
|
||||||
|
const MD_FILE = OUT_DIR + '/all-releases.md';
|
||||||
|
|
||||||
|
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||||
|
|
||||||
|
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
||||||
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||||
|
const EXTRA_HEADERS = {
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
'Accept-Encoding': 'gzip, deflate, br'
|
||||||
|
};
|
||||||
|
|
||||||
|
async function setupPage(browser) {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.setUserAgent(USER_AGENT);
|
||||||
|
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAllUrls() {
|
||||||
|
if (fs.existsSync(URLS_FILE)) {
|
||||||
|
const urls = JSON.parse(fs.readFileSync(URLS_FILE, 'utf8'));
|
||||||
|
console.log(`📋 ${urls.length} URLs (cached)`);
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||||
|
await new Promise(r => setTimeout(r, 3000));
|
||||||
|
|
||||||
|
const baseUrl = new URL(BASE);
|
||||||
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
|
const allLinks = await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
|
||||||
|
const seen = new Set();
|
||||||
|
const docPages = [];
|
||||||
|
for (const link of allLinks) {
|
||||||
|
try {
|
||||||
|
const u = new URL(link.href);
|
||||||
|
if (u.hostname !== baseUrl.hostname) continue;
|
||||||
|
if (!u.pathname.startsWith(articleDir)) continue;
|
||||||
|
if (u.hash) continue;
|
||||||
|
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
||||||
|
const fn = u.pathname.split('/').pop();
|
||||||
|
if (!fn || fn.length < 3) continue;
|
||||||
|
if (!seen.has(fn)) {
|
||||||
|
seen.add(fn);
|
||||||
|
docPages.push({ href: link.href, text: link.text });
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
docPages.unshift({ href: BASE, text: 'All Releases (Index)' });
|
||||||
|
fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
|
||||||
|
console.log(`📋 ${docPages.length} URLs found`);
|
||||||
|
return docPages;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractPage(page, href) {
|
||||||
|
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 600));
|
||||||
|
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.MCWebHelpFramesetLink'];
|
||||||
|
let main = document.body;
|
||||||
|
for (const sel of mainSel) {
|
||||||
|
const el = document.querySelector(sel);
|
||||||
|
if (el) { main = el; break; }
|
||||||
|
}
|
||||||
|
const h1 = main.querySelector('h1, h2');
|
||||||
|
const title = h1 ? h1.textContent.trim() : document.title.trim();
|
||||||
|
|
||||||
|
const blocks = [];
|
||||||
|
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
||||||
|
const txt = el.textContent.trim();
|
||||||
|
if (!txt || txt.length < 3) return;
|
||||||
|
const tag = el.tagName;
|
||||||
|
if (tag.match(/^H[1-4]$/)) {
|
||||||
|
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
||||||
|
} else if (tag === 'LI') {
|
||||||
|
blocks.push('- ' + txt);
|
||||||
|
} else if (tag === 'PRE' || tag === 'CODE') {
|
||||||
|
blocks.push('\n```\n' + txt + '\n```\n');
|
||||||
|
} else if (tag === 'TABLE') {
|
||||||
|
const rows = [];
|
||||||
|
el.querySelectorAll('tr').forEach(tr => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
||||||
|
if (cells) rows.push('| ' + cells + ' |');
|
||||||
|
});
|
||||||
|
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
||||||
|
} else {
|
||||||
|
blocks.push(txt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return { title, body: blocks.join('\n\n') };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function crawl() {
|
||||||
|
const urls = await getAllUrls();
|
||||||
|
console.log(`\n📥 Starting crawl of ${urls.length} pages...\n`);
|
||||||
|
|
||||||
|
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes\n\n> Source: ${BASE}\n> Pages: ${urls.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
||||||
|
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
|
||||||
|
let ok = 0, fail = 0;
|
||||||
|
for (let i = 0; i < urls.length; i++) {
|
||||||
|
const { href, text } = urls[i];
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, href);
|
||||||
|
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
ok++;
|
||||||
|
console.log(`✅ [${i+1}/${urls.length}] ${content.title.slice(0, 60).replace(/\n/g, ' ')}`);
|
||||||
|
} catch (e) {
|
||||||
|
fail++;
|
||||||
|
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${urls.length}] ${e.message.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
console.log(`\n📊 Done: ${ok} OK, ${fail} failed`);
|
||||||
|
console.log(`📁 Output: ${MD_FILE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
crawl().catch(err => { console.error(err); process.exit(1); });
|
||||||
+251
@@ -0,0 +1,251 @@
|
|||||||
|
// Crawl HPE Aruba Networking AOS 10.x Release Notes - v2
|
||||||
|
// Phase 1: discover all version overview pages
|
||||||
|
// Phase 2: for each version, fetch features/resolved/known sub-pages
|
||||||
|
const puppeteer = require('puppeteer-core');
|
||||||
|
const fs = require('fs');
|
||||||
|
|
||||||
|
// Auto-detect Chrome across Mac / Linux / Windows
|
||||||
|
function findChrome() {
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const homedir = require('os').homedir();
|
||||||
|
const candidates = [
|
||||||
|
// macOS (local dev)
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.167/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
|
||||||
|
// Linux
|
||||||
|
'/usr/bin/google-chrome',
|
||||||
|
'/usr/bin/chromium-browser',
|
||||||
|
'/usr/bin/chromium',
|
||||||
|
// Docker puppeteer cache
|
||||||
|
path.join(homedir, '.cache/puppeteer/chrome/'),
|
||||||
|
];
|
||||||
|
for (const c of candidates) {
|
||||||
|
if (fs.existsSync(c)) return c;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const CHROME_PATH = findChrome();
|
||||||
|
if (!CHROME_PATH) {
|
||||||
|
console.error('No Chrome found. Try: apt install chromium / google-chrome-stable');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log('Using Chrome:', CHROME_PATH);
|
||||||
|
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
|
||||||
|
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
|
||||||
|
const VERSIONS_FILE = OUT_DIR + '/versions.json';
|
||||||
|
const MD_FILE = OUT_DIR + '/all-releases-v2.md';
|
||||||
|
|
||||||
|
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||||
|
|
||||||
|
const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
|
||||||
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
|
||||||
|
const EXTRA_HEADERS = {
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9',
|
||||||
|
'Accept-Encoding': 'gzip, deflate, br'
|
||||||
|
};
|
||||||
|
|
||||||
|
const SUB_PAGE_PATTERNS = [
|
||||||
|
{ regex: /features[/-]\d+\.htm/i, type: 'features' },
|
||||||
|
{ regex: /resolved-issues[/-]\d+\.htm/i, type: 'resolved' },
|
||||||
|
{ regex: /known-issues[/-]\d+\.htm/i, type: 'known' },
|
||||||
|
{ regex: /regulatory[/-]\d+\.htm/i, type: 'regulatory' },
|
||||||
|
];
|
||||||
|
|
||||||
|
async function setupPage(browser) {
|
||||||
|
const page = await browser.newPage();
|
||||||
|
await page.setUserAgent(USER_AGENT);
|
||||||
|
await page.setExtraHTTPHeaders(EXTRA_HEADERS);
|
||||||
|
// Skip script/style in extraction
|
||||||
|
await page.evaluateOnNewDocument(() => {
|
||||||
|
window.MutationObserver = window.MutationObserver || undefined;
|
||||||
|
});
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean DOM extraction: skip script/style blocks
|
||||||
|
async function extractPage(page, href) {
|
||||||
|
await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 600));
|
||||||
|
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
// Remove script + style + nav elements from clone
|
||||||
|
const document2 = document.cloneNode(true);
|
||||||
|
document2.querySelectorAll('script, style, nav, .MCWebHelpFramesetLink, .toolbar, .navigation').forEach(el => el.remove());
|
||||||
|
|
||||||
|
const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.body-container'];
|
||||||
|
let main = document2.body;
|
||||||
|
for (const sel of mainSel) {
|
||||||
|
const el = document2.querySelector(sel);
|
||||||
|
if (el) { main = el; break; }
|
||||||
|
}
|
||||||
|
|
||||||
|
const h1 = main.querySelector('h1, h2');
|
||||||
|
const title = h1 ? h1.textContent.trim() : document2.title.trim();
|
||||||
|
|
||||||
|
const blocks = [];
|
||||||
|
main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
|
||||||
|
const txt = el.textContent.trim();
|
||||||
|
if (!txt || txt.length < 3) return;
|
||||||
|
const tag = el.tagName;
|
||||||
|
if (tag.match(/^H[1-4]$/)) {
|
||||||
|
blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
|
||||||
|
} else if (tag === 'LI') {
|
||||||
|
blocks.push('- ' + txt);
|
||||||
|
} else if (tag === 'PRE' || tag === 'CODE') {
|
||||||
|
blocks.push('\n```\n' + txt + '\n```\n');
|
||||||
|
} else if (tag === 'TABLE') {
|
||||||
|
const rows = [];
|
||||||
|
el.querySelectorAll('tr').forEach(tr => {
|
||||||
|
const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
|
||||||
|
if (cells) rows.push('| ' + cells + ' |');
|
||||||
|
});
|
||||||
|
if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
|
||||||
|
} else {
|
||||||
|
blocks.push(txt);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return { title, body: blocks.join('\n\n') };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverVersions(page) {
|
||||||
|
if (fs.existsSync(VERSIONS_FILE)) {
|
||||||
|
const versions = JSON.parse(fs.readFileSync(VERSIONS_FILE, 'utf8'));
|
||||||
|
console.log(`📋 ${versions.length} versions (cached)`);
|
||||||
|
return versions;
|
||||||
|
}
|
||||||
|
await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||||
|
await new Promise(r => setTimeout(r, 3000));
|
||||||
|
|
||||||
|
const baseUrl = new URL(BASE);
|
||||||
|
const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);
|
||||||
|
|
||||||
|
const allLinks = await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
|
||||||
|
const seen = new Set();
|
||||||
|
const versions = [];
|
||||||
|
for (const link of allLinks) {
|
||||||
|
try {
|
||||||
|
const u = new URL(link.href);
|
||||||
|
if (u.hostname !== baseUrl.hostname) continue;
|
||||||
|
if (!u.pathname.startsWith(articleDir)) continue;
|
||||||
|
if (u.hash) continue;
|
||||||
|
if (/javascript:|^#|^\?/i.test(link.href)) continue;
|
||||||
|
const fn = u.pathname.split('/').pop();
|
||||||
|
if (!fn || fn.length < 3) continue;
|
||||||
|
if (!seen.has(fn)) {
|
||||||
|
seen.add(fn);
|
||||||
|
versions.push({ href: link.href, text: link.text });
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versions, null, 2));
|
||||||
|
console.log(`📋 ${versions.length} version pages found`);
|
||||||
|
return versions;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function discoverSubPages(page, versionUrl) {
|
||||||
|
await page.goto(versionUrl, { waitUntil: 'networkidle2', timeout: 30000 });
|
||||||
|
await new Promise(r => setTimeout(r, 1500));
|
||||||
|
return await page.evaluate(() => {
|
||||||
|
const links = [];
|
||||||
|
document.querySelectorAll('a[href]').forEach(a => {
|
||||||
|
const href = a.href;
|
||||||
|
const text = a.textContent.trim();
|
||||||
|
if (/features|resolved-issues|known-issues|regulatory|whats-new/.test(href) && !/^#/.test(href) && !/Send Feedback/.test(text)) {
|
||||||
|
links.push({ href, text: text.slice(0, 80) });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return links;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function crawl() {
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
executablePath: CHROME_PATH,
|
||||||
|
headless: true,
|
||||||
|
args: CHROME_ARGS
|
||||||
|
});
|
||||||
|
const page = await setupPage(browser);
|
||||||
|
|
||||||
|
const versions = await discoverVersions(page);
|
||||||
|
console.log(`\n📥 Phase 1 (${versions.length} version overviews)...\n`);
|
||||||
|
|
||||||
|
fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes (v2)\n\n> Source: ${BASE}\n> Versions: ${versions.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);
|
||||||
|
|
||||||
|
let ok = 0, fail = 0;
|
||||||
|
const allSubPages = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < versions.length; i++) {
|
||||||
|
const { href, text } = versions[i];
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, href);
|
||||||
|
const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
ok++;
|
||||||
|
console.log(`✅ [${i+1}/${versions.length}] ${content.title.slice(0, 50).replace(/\n/g, ' ')}`);
|
||||||
|
|
||||||
|
// Phase 2: discover sub-pages for this version
|
||||||
|
const subPages = await discoverSubPages(page, href);
|
||||||
|
for (const sub of subPages) {
|
||||||
|
// Dedup: only add sub-pages for this specific version
|
||||||
|
if (sub.href.includes(href.split('/').slice(0, -1).join('/')) || sub.href.includes('/10.')) {
|
||||||
|
allSubPages.push({ parent: text, parentHref: href, ...sub });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
fail++;
|
||||||
|
const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${versions.length}] ${e.message.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dedupe sub-pages
|
||||||
|
const seenSub = new Set();
|
||||||
|
const uniqueSub = allSubPages.filter(s => {
|
||||||
|
if (seenSub.has(s.href)) return false;
|
||||||
|
seenSub.add(s.href);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`\n📥 Phase 2 (${uniqueSub.length} sub-pages: features/resolved/known)...\n`);
|
||||||
|
|
||||||
|
fs.appendFileSync(MD_FILE, `\n\n# === Detail Sub-Pages (Features / Resolved Issues / Known Issues) ===\n\n`);
|
||||||
|
|
||||||
|
let subOk = 0, subFail = 0;
|
||||||
|
for (let i = 0; i < uniqueSub.length; i++) {
|
||||||
|
const sub = uniqueSub[i];
|
||||||
|
const type = SUB_PAGE_PATTERNS.find(p => p.regex.test(sub.href))?.type || 'detail';
|
||||||
|
try {
|
||||||
|
const content = await extractPage(page, sub.href);
|
||||||
|
const md = `## ${sub.parent} — ${type.toUpperCase()}\n\n_${sub.text}_\n\n_Source: ${sub.href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
subOk++;
|
||||||
|
console.log(`✅ [${i+1}/${uniqueSub.length}] ${type.padEnd(10)} ${sub.parent} — ${sub.text.slice(0, 40)}`);
|
||||||
|
} catch (e) {
|
||||||
|
subFail++;
|
||||||
|
const md = `## ${sub.parent} — ${type}\n\n_❌ Error: ${e.message}_\n\n_Source: ${sub.href}_\n\n---\n\n`;
|
||||||
|
fs.appendFileSync(MD_FILE, md);
|
||||||
|
console.log(`❌ [${i+1}/${uniqueSub.length}] ${type} ${e.message.slice(0, 60)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
|
console.log(`\n📊 Done:`);
|
||||||
|
console.log(` Phase 1 (overviews): ${ok} OK, ${fail} failed`);
|
||||||
|
console.log(` Phase 2 (sub-pages): ${subOk} OK, ${subFail} failed`);
|
||||||
|
console.log(`📁 Output: ${MD_FILE}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
crawl().catch(err => { console.error(err); process.exit(1); });
|
||||||
Generated
+205
-7
@@ -9,7 +9,8 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"puppeteer": "^25.0.4"
|
"puppeteer": "^25.0.4",
|
||||||
|
"puppeteer-core": "^25.3.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@babel/code-frame": {
|
"node_modules/@babel/code-frame": {
|
||||||
@@ -369,6 +370,18 @@
|
|||||||
"node": "6.* || 8.* || >= 10.*"
|
"node": "6.* || 8.* || >= 10.*"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/get-east-asian-width": {
|
||||||
|
"version": "1.6.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz",
|
||||||
|
"integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/import-fresh": {
|
"node_modules/import-fresh": {
|
||||||
"version": "3.3.1",
|
"version": "3.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
|
||||||
@@ -436,6 +449,15 @@
|
|||||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
|
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/modern-tar": {
|
||||||
|
"version": "0.7.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.7.7.tgz",
|
||||||
|
"integrity": "sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/ms": {
|
"node_modules/ms": {
|
||||||
"version": "2.1.3",
|
"version": "2.1.3",
|
||||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||||
@@ -528,6 +550,176 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/puppeteer-core": {
|
"node_modules/puppeteer-core": {
|
||||||
|
"version": "25.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.3.0.tgz",
|
||||||
|
"integrity": "sha512-fm+wpUr2oigH1PXZvwgATrM2tYWHMDG8ASzTEe9uukCye4X5Ldx1K5BTHPFKITrIWvQQAQ256d1NpbEveBcKjA==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@puppeteer/browsers": "3.0.6",
|
||||||
|
"chromium-bidi": "16.0.1",
|
||||||
|
"devtools-protocol": "0.0.1638949",
|
||||||
|
"typed-query-selector": "^2.12.2",
|
||||||
|
"webdriver-bidi-protocol": "0.4.2",
|
||||||
|
"ws": "^8.21.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=22.12.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/@puppeteer/browsers": {
|
||||||
|
"version": "3.0.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.6.tgz",
|
||||||
|
"integrity": "sha512-B/gKoqlFkzhvzsI6jo9K1cZz9o5ypviVv/xu8CwA4grZzyVwN+XfkT+tu8T1zrauuEXv6VhS2oGX+6NL95WcKA==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"modern-tar": "^0.7.6",
|
||||||
|
"yargs": "^18.0.0"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"browsers": "lib/main-cli.js"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=22.12.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"proxy-agent": ">=8.0.1",
|
||||||
|
"yauzl": "^2.10.0 || ^3.4.0"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"proxy-agent": {
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"yauzl": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/ansi-regex": {
|
||||||
|
"version": "6.2.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
|
||||||
|
"integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/ansi-styles": {
|
||||||
|
"version": "6.2.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz",
|
||||||
|
"integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/cliui": {
|
||||||
|
"version": "9.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz",
|
||||||
|
"integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"string-width": "^7.2.0",
|
||||||
|
"strip-ansi": "^7.1.0",
|
||||||
|
"wrap-ansi": "^9.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/devtools-protocol": {
|
||||||
|
"version": "0.0.1638949",
|
||||||
|
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1638949.tgz",
|
||||||
|
"integrity": "sha512-mXwg4Fqnv0WR4iuAT/gYUmctNkjILwXFHyZ+m7Ty1dfr0ezZt2U3gnrrJTfRobJTHoXf+IbuFvFITzLrLFjwJA==",
|
||||||
|
"license": "BSD-3-Clause"
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/emoji-regex": {
|
||||||
|
"version": "10.6.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz",
|
||||||
|
"integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/string-width": {
|
||||||
|
"version": "7.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
|
||||||
|
"integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"emoji-regex": "^10.3.0",
|
||||||
|
"get-east-asian-width": "^1.0.0",
|
||||||
|
"strip-ansi": "^7.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/sponsors/sindresorhus"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/strip-ansi": {
|
||||||
|
"version": "7.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
|
||||||
|
"integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"ansi-regex": "^6.2.2"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/wrap-ansi": {
|
||||||
|
"version": "9.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz",
|
||||||
|
"integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"ansi-styles": "^6.2.1",
|
||||||
|
"string-width": "^7.0.0",
|
||||||
|
"strip-ansi": "^7.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/yargs": {
|
||||||
|
"version": "18.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz",
|
||||||
|
"integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==",
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"cliui": "^9.0.1",
|
||||||
|
"escalade": "^3.1.1",
|
||||||
|
"get-caller-file": "^2.0.5",
|
||||||
|
"string-width": "^7.2.0",
|
||||||
|
"y18n": "^5.0.5",
|
||||||
|
"yargs-parser": "^22.0.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": "^20.19.0 || ^22.12.0 || >=23"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer-core/node_modules/yargs-parser": {
|
||||||
|
"version": "22.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz",
|
||||||
|
"integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==",
|
||||||
|
"license": "ISC",
|
||||||
|
"engines": {
|
||||||
|
"node": "^20.19.0 || ^22.12.0 || >=23"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/puppeteer/node_modules/puppeteer-core": {
|
||||||
"version": "25.0.4",
|
"version": "25.0.4",
|
||||||
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.0.4.tgz",
|
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.0.4.tgz",
|
||||||
"integrity": "sha512-K1LQKDP6w1rIr1jUyN9obH16TO/DCy86k3q+FBd2prGY+TStxhFySxmaZZuRF+0D3BJXjwCYFke7tMHCH4olTA==",
|
"integrity": "sha512-K1LQKDP6w1rIr1jUyN9obH16TO/DCy86k3q+FBd2prGY+TStxhFySxmaZZuRF+0D3BJXjwCYFke7tMHCH4olTA==",
|
||||||
@@ -545,6 +737,12 @@
|
|||||||
"node": ">=22.12.0"
|
"node": ">=22.12.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/puppeteer/node_modules/webdriver-bidi-protocol": {
|
||||||
|
"version": "0.4.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
||||||
|
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
||||||
|
"license": "Apache-2.0"
|
||||||
|
},
|
||||||
"node_modules/require-directory": {
|
"node_modules/require-directory": {
|
||||||
"version": "2.1.1",
|
"version": "2.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||||
@@ -663,9 +861,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/webdriver-bidi-protocol": {
|
"node_modules/webdriver-bidi-protocol": {
|
||||||
"version": "0.4.1",
|
"version": "0.4.2",
|
||||||
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz",
|
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.2.tgz",
|
||||||
"integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==",
|
"integrity": "sha512-VSV+fzfChirL3e7jay2yUC7B4HQCGtEWEg/MSSQbK+qWbqeGlRLlXTzPpYr3XGUvbpDHumWZBJxgesg4N7dbtA==",
|
||||||
"license": "Apache-2.0"
|
"license": "Apache-2.0"
|
||||||
},
|
},
|
||||||
"node_modules/wrap-ansi": {
|
"node_modules/wrap-ansi": {
|
||||||
@@ -692,9 +890,9 @@
|
|||||||
"license": "ISC"
|
"license": "ISC"
|
||||||
},
|
},
|
||||||
"node_modules/ws": {
|
"node_modules/ws": {
|
||||||
"version": "8.20.1",
|
"version": "8.21.1",
|
||||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz",
|
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz",
|
||||||
"integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==",
|
"integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=10.0.0"
|
"node": ">=10.0.0"
|
||||||
|
|||||||
+2
-1
@@ -10,6 +10,7 @@
|
|||||||
"author": "",
|
"author": "",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"puppeteer": "^25.0.4"
|
"puppeteer": "^25.0.4",
|
||||||
|
"puppeteer-core": "^25.3.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user