diff --git a/crawl_hpe.js b/crawl_hpe.js new file mode 100644 index 0000000..a093c57 --- /dev/null +++ b/crawl_hpe.js @@ -0,0 +1,158 @@ +// Crawl HPE Aruba Networking AOS 10.x Release Notes +// Uses puppeteer-core + existing Chrome for Testing +const puppeteer = require('puppeteer-core'); +const fs = require('fs'); + +const CHROME_PATH = '/Users/admin/.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'; +const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm'; +const OUT_DIR = '/tmp/hpe-aruba-aos10-rn'; +const URLS_FILE = OUT_DIR + '/urls.json'; +const MD_FILE = OUT_DIR + '/all-releases.md'; + +if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true }); + +const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled']; +const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'; +const EXTRA_HEADERS = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br' +}; + +async function setupPage(browser) { + const page = await browser.newPage(); + await page.setUserAgent(USER_AGENT); + await page.setExtraHTTPHeaders(EXTRA_HEADERS); + return page; +} + +async function getAllUrls() { + if (fs.existsSync(URLS_FILE)) { + const urls = JSON.parse(fs.readFileSync(URLS_FILE, 'utf8')); + console.log(`πŸ“‹ ${urls.length} URLs (cached)`); + return urls; + } + + const browser = await puppeteer.launch({ + executablePath: CHROME_PATH, + headless: true, + args: CHROME_ARGS + }); + const page = await setupPage(browser); + await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 }); + await new Promise(r => setTimeout(r, 3000)); + + const baseUrl = new URL(BASE); + const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1); + + const allLinks = await page.evaluate(() => { + const links = []; + document.querySelectorAll('a[href]').forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) }); + }); + return links; + }); + + await browser.close(); + + const seen = new Set(); + const docPages = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (u.hash) continue; + if (/javascript:|^#|^\?/i.test(link.href)) continue; + const fn = u.pathname.split('/').pop(); + if (!fn || fn.length < 3) continue; + if (!seen.has(fn)) { + seen.add(fn); + docPages.push({ href: link.href, text: link.text }); + } + } catch (e) {} + } + + docPages.unshift({ href: BASE, text: 'All Releases (Index)' }); + fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2)); + console.log(`πŸ“‹ ${docPages.length} URLs found`); + return docPages; +} + +async function extractPage(page, href) { + await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 }); + await new Promise(r => setTimeout(r, 600)); + + return await page.evaluate(() => { + const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.MCWebHelpFramesetLink']; + let main = document.body; + for (const sel of mainSel) { + const el = document.querySelector(sel); + if (el) { main = el; break; } + } + const h1 = main.querySelector('h1, h2'); + const title = h1 ? h1.textContent.trim() : document.title.trim(); + + const blocks = []; + main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => { + const txt = el.textContent.trim(); + if (!txt || txt.length < 3) return; + const tag = el.tagName; + if (tag.match(/^H[1-4]$/)) { + blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n'); + } else if (tag === 'LI') { + blocks.push('- ' + txt); + } else if (tag === 'PRE' || tag === 'CODE') { + blocks.push('\n```\n' + txt + '\n```\n'); + } else if (tag === 'TABLE') { + const rows = []; + el.querySelectorAll('tr').forEach(tr => { + const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | '); + if (cells) rows.push('| ' + cells + ' |'); + }); + if (rows.length) blocks.push('\n' + rows.join('\n') + '\n'); + } else { + blocks.push(txt); + } + }); + return { title, body: blocks.join('\n\n') }; + }); +} + +async function crawl() { + const urls = await getAllUrls(); + console.log(`\nπŸ“₯ Starting crawl of ${urls.length} pages...\n`); + + fs.writeFileSync(MD_FILE, `# HPE Aruba Networking β€” AOS 10.x Release Notes\n\n> Source: ${BASE}\n> Pages: ${urls.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`); + + const browser = await puppeteer.launch({ + executablePath: CHROME_PATH, + headless: true, + args: CHROME_ARGS + }); + const page = await setupPage(browser); + + let ok = 0, fail = 0; + for (let i = 0; i < urls.length; i++) { + const { href, text } = urls[i]; + try { + const content = await extractPage(page, href); + const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + ok++; + console.log(`βœ… [${i+1}/${urls.length}] ${content.title.slice(0, 60).replace(/\n/g, ' ')}`); + } catch (e) { + fail++; + const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + console.log(`❌ [${i+1}/${urls.length}] ${e.message.slice(0, 80)}`); + } + } + + await browser.close(); + console.log(`\nπŸ“Š Done: ${ok} OK, ${fail} failed`); + console.log(`πŸ“ Output: ${MD_FILE}`); +} + +crawl().catch(err => { console.error(err); process.exit(1); }); diff --git a/crawl_hpe_v2.js b/crawl_hpe_v2.js new file mode 100644 index 0000000..bf683e7 --- /dev/null +++ b/crawl_hpe_v2.js @@ -0,0 +1,251 @@ +// Crawl HPE Aruba Networking AOS 10.x Release Notes - v2 +// Phase 1: discover all version overview pages +// Phase 2: for each version, fetch features/resolved/known sub-pages +const puppeteer = require('puppeteer-core'); +const fs = require('fs'); + +// Auto-detect Chrome across Mac / Linux / Windows +function findChrome() { + const fs = require('fs'); + const path = require('path'); + const homedir = require('os').homedir(); + const candidates = [ + // macOS (local dev) + path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'), + path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.167/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'), + // Linux + '/usr/bin/google-chrome', + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + // Docker puppeteer cache + path.join(homedir, '.cache/puppeteer/chrome/'), + ]; + for (const c of candidates) { + if (fs.existsSync(c)) return c; + } + return null; +} +const CHROME_PATH = findChrome(); +if (!CHROME_PATH) { + console.error('No Chrome found. Try: apt install chromium / google-chrome-stable'); + process.exit(1); +} +console.log('Using Chrome:', CHROME_PATH); +const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm'; +const OUT_DIR = '/tmp/hpe-aruba-aos10-rn'; +const VERSIONS_FILE = OUT_DIR + '/versions.json'; +const MD_FILE = OUT_DIR + '/all-releases-v2.md'; + +if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true }); + +const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled']; +const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'; +const EXTRA_HEADERS = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br' +}; + +const SUB_PAGE_PATTERNS = [ + { regex: /features[/-]\d+\.htm/i, type: 'features' }, + { regex: /resolved-issues[/-]\d+\.htm/i, type: 'resolved' }, + { regex: /known-issues[/-]\d+\.htm/i, type: 'known' }, + { regex: /regulatory[/-]\d+\.htm/i, type: 'regulatory' }, +]; + +async function setupPage(browser) { + const page = await browser.newPage(); + await page.setUserAgent(USER_AGENT); + await page.setExtraHTTPHeaders(EXTRA_HEADERS); + // Skip script/style in extraction + await page.evaluateOnNewDocument(() => { + window.MutationObserver = window.MutationObserver || undefined; + }); + return page; +} + +// Clean DOM extraction: skip script/style blocks +async function extractPage(page, href) { + await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 }); + await new Promise(r => setTimeout(r, 600)); + + return await page.evaluate(() => { + // Remove script + style + nav elements from clone + const document2 = document.cloneNode(true); + document2.querySelectorAll('script, style, nav, .MCWebHelpFramesetLink, .toolbar, .navigation').forEach(el => el.remove()); + + const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.body-container']; + let main = document2.body; + for (const sel of mainSel) { + const el = document2.querySelector(sel); + if (el) { main = el; break; } + } + + const h1 = main.querySelector('h1, h2'); + const title = h1 ? h1.textContent.trim() : document2.title.trim(); + + const blocks = []; + main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => { + const txt = el.textContent.trim(); + if (!txt || txt.length < 3) return; + const tag = el.tagName; + if (tag.match(/^H[1-4]$/)) { + blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n'); + } else if (tag === 'LI') { + blocks.push('- ' + txt); + } else if (tag === 'PRE' || tag === 'CODE') { + blocks.push('\n```\n' + txt + '\n```\n'); + } else if (tag === 'TABLE') { + const rows = []; + el.querySelectorAll('tr').forEach(tr => { + const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | '); + if (cells) rows.push('| ' + cells + ' |'); + }); + if (rows.length) blocks.push('\n' + rows.join('\n') + '\n'); + } else { + blocks.push(txt); + } + }); + return { title, body: blocks.join('\n\n') }; + }); +} + +async function discoverVersions(page) { + if (fs.existsSync(VERSIONS_FILE)) { + const versions = JSON.parse(fs.readFileSync(VERSIONS_FILE, 'utf8')); + console.log(`πŸ“‹ ${versions.length} versions (cached)`); + return versions; + } + await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 }); + await new Promise(r => setTimeout(r, 3000)); + + const baseUrl = new URL(BASE); + const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1); + + const allLinks = await page.evaluate(() => { + const links = []; + document.querySelectorAll('a[href]').forEach(a => { + const text = a.textContent.trim(); + if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) }); + }); + return links; + }); + + const seen = new Set(); + const versions = []; + for (const link of allLinks) { + try { + const u = new URL(link.href); + if (u.hostname !== baseUrl.hostname) continue; + if (!u.pathname.startsWith(articleDir)) continue; + if (u.hash) continue; + if (/javascript:|^#|^\?/i.test(link.href)) continue; + const fn = u.pathname.split('/').pop(); + if (!fn || fn.length < 3) continue; + if (!seen.has(fn)) { + seen.add(fn); + versions.push({ href: link.href, text: link.text }); + } + } catch (e) {} + } + + fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versions, null, 2)); + console.log(`πŸ“‹ ${versions.length} version pages found`); + return versions; +} + +async function discoverSubPages(page, versionUrl) { + await page.goto(versionUrl, { waitUntil: 'networkidle2', timeout: 30000 }); + await new Promise(r => setTimeout(r, 1500)); + return await page.evaluate(() => { + const links = []; + document.querySelectorAll('a[href]').forEach(a => { + const href = a.href; + const text = a.textContent.trim(); + if (/features|resolved-issues|known-issues|regulatory|whats-new/.test(href) && !/^#/.test(href) && !/Send Feedback/.test(text)) { + links.push({ href, text: text.slice(0, 80) }); + } + }); + return links; + }); +} + +async function crawl() { + const browser = await puppeteer.launch({ + executablePath: CHROME_PATH, + headless: true, + args: CHROME_ARGS + }); + const page = await setupPage(browser); + + const versions = await discoverVersions(page); + console.log(`\nπŸ“₯ Phase 1 (${versions.length} version overviews)...\n`); + + fs.writeFileSync(MD_FILE, `# HPE Aruba Networking β€” AOS 10.x Release Notes (v2)\n\n> Source: ${BASE}\n> Versions: ${versions.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`); + + let ok = 0, fail = 0; + const allSubPages = []; + + for (let i = 0; i < versions.length; i++) { + const { href, text } = versions[i]; + try { + const content = await extractPage(page, href); + const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + ok++; + console.log(`βœ… [${i+1}/${versions.length}] ${content.title.slice(0, 50).replace(/\n/g, ' ')}`); + + // Phase 2: discover sub-pages for this version + const subPages = await discoverSubPages(page, href); + for (const sub of subPages) { + // Dedup: only add sub-pages for this specific version + if (sub.href.includes(href.split('/').slice(0, -1).join('/')) || sub.href.includes('/10.')) { + allSubPages.push({ parent: text, parentHref: href, ...sub }); + } + } + } catch (e) { + fail++; + const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + console.log(`❌ [${i+1}/${versions.length}] ${e.message.slice(0, 80)}`); + } + } + + // Dedupe sub-pages + const seenSub = new Set(); + const uniqueSub = allSubPages.filter(s => { + if (seenSub.has(s.href)) return false; + seenSub.add(s.href); + return true; + }); + + console.log(`\nπŸ“₯ Phase 2 (${uniqueSub.length} sub-pages: features/resolved/known)...\n`); + + fs.appendFileSync(MD_FILE, `\n\n# === Detail Sub-Pages (Features / Resolved Issues / Known Issues) ===\n\n`); + + let subOk = 0, subFail = 0; + for (let i = 0; i < uniqueSub.length; i++) { + const sub = uniqueSub[i]; + const type = SUB_PAGE_PATTERNS.find(p => p.regex.test(sub.href))?.type || 'detail'; + try { + const content = await extractPage(page, sub.href); + const md = `## ${sub.parent} β€” ${type.toUpperCase()}\n\n_${sub.text}_\n\n_Source: ${sub.href}_\n\n${content.body || '_no content_'}\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + subOk++; + console.log(`βœ… [${i+1}/${uniqueSub.length}] ${type.padEnd(10)} ${sub.parent} β€” ${sub.text.slice(0, 40)}`); + } catch (e) { + subFail++; + const md = `## ${sub.parent} β€” ${type}\n\n_❌ Error: ${e.message}_\n\n_Source: ${sub.href}_\n\n---\n\n`; + fs.appendFileSync(MD_FILE, md); + console.log(`❌ [${i+1}/${uniqueSub.length}] ${type} ${e.message.slice(0, 60)}`); + } + } + + await browser.close(); + console.log(`\nπŸ“Š Done:`); + console.log(` Phase 1 (overviews): ${ok} OK, ${fail} failed`); + console.log(` Phase 2 (sub-pages): ${subOk} OK, ${subFail} failed`); + console.log(`πŸ“ Output: ${MD_FILE}`); +} + +crawl().catch(err => { console.error(err); process.exit(1); }); diff --git a/hpe-aos10-versions.json b/hpe-aos10-versions.json new file mode 100644 index 0000000..6331776 --- /dev/null +++ b/hpe-aos10-versions.json @@ -0,0 +1,242 @@ +[ + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/Home.htm", + "text": "Home" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/common%20files/topic_files/whats_new.htm", + "text": "What's New" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm", + "text": "All Releases" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/common%20files/topic_files/terminology-change.htm", + "text": "Terminology Change" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/overview-1031x.htm", + "text": "10.3.1.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/overview-1040x.htm", + "text": "10.4.0.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/overview-1041x.htm", + "text": "10.4.1.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/overview-1050x.htm", + "text": "10.5.0.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/overview-1051x.htm", + "text": "10.5.1.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/overview-1060x.htm", + "text": "10.6.0.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/overview-1070x.htm", + "text": "10.7.0.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/overview-1071x.htm", + "text": "10.7.1.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/overview-1072x.htm", + "text": "10.7.2.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/overview-108x.htm", + "text": "10.8.x.x" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/10/overview-10810.htm", + "text": "10.8.1.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/02/overview-10802.htm", + "text": "10.8.0.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/01/overview-10801.htm", + "text": "10.8.0.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.8/00/overview-10800.htm", + "text": "10.8.0.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/6/overview-10726.htm", + "text": "10.7.2.6" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/5/overview-10725.htm", + "text": "10.7.2.5" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/4/overview-10724.htm", + "text": "10.7.2.4" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/3/overview-10723.htm", + "text": "10.7.2.3" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/2/overview-10722.htm", + "text": "10.7.2.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/1/overview-10721.htm", + "text": "10.7.2.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.2/0/overview-10720.htm", + "text": "10.7.2.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/1/overview-10711.htm", + "text": "10.7.1.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7.1/0/overview-10710.htm", + "text": "10.7.1.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/02/overview-10702.htm", + "text": "10.7.0.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/01/overview-10701.htm", + "text": "10.7.0.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.7/00/overview-10700.htm", + "text": "10.7.0.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/03/overview-10603.htm", + "text": "10.6.0.3" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/02/overview-10602.htm", + "text": "10.6.0.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/01/overview-10601.htm", + "text": "10.6.0.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.6/00/overview-10600.htm", + "text": "10.6.0.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/1/overview-10511.htm", + "text": "10.5.1.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5.1/0/overview-10510.htm", + "text": "10.5.1.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/01/overview-10501.htm", + "text": "10.5.0.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.5/00/overview-10500.htm", + "text": "10.5.0.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/12/overview-104112.htm", + "text": "10.4.1.12" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/11/overview-104111.htm", + "text": "10.4.1.11" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/10/overview-104110.htm", + "text": "10.4.1.10" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/9/overview-10419.htm", + "text": "10.4.1.9" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/8/overview-10418.htm", + "text": "10.4.1.8" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/7/overview-10417.htm", + "text": "10.4.1.7" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/6/overview-10416.htm", + "text": "10.4.1.6" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/5/overview-10415.htm", + "text": "10.4.1.5" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/4/overview-10414.htm", + "text": "10.4.1.4" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/3/overview-10413.htm", + "text": "10.4.1.3" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/2/overview-10412.htm", + "text": "10.4.1.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/1/overview-10411.htm", + "text": "10.4.1.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4.1/0/overview-10410.htm", + "text": "10.4.1.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/03/overview-10403.htm", + "text": "10.4.0.3" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/02/overview-10402.htm", + "text": "10.4.0.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/01/overview-10401.htm", + "text": "10.4.0.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.4/00/overview-10400.htm", + "text": "10.4.0.0" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/4/overview-10314.htm", + "text": "10.3.1.4" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/3/overview-10313.htm", + "text": "10.3.1.3" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/2/overview-10312.htm", + "text": "10.3.1.2" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/1/overview-10311.htm", + "text": "10.3.1.1" + }, + { + "href": "https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/10.3.1/0/overview-10310.htm", + "text": "10.3.1.0" + } +] \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..820ba05 --- /dev/null +++ b/index.html @@ -0,0 +1,142 @@ + + + + + +URL-to-PDF Β· Doc Crawler + + + +
+

πŸ“„ URL-to-PDF Β· Doc Crawler

+

Paste a documentation URL β†’ get all sub-pages as Markdown

+ +
+ πŸ• ITη‹— 2026-07-24 β€” Two-phase crawler: discovers version pages, then fetches each version's features / resolved / known issues / regulatory sub-pages. Currently tested with HPE AOS 10.x (209 pages). +
+ + v2Two-phaseMarkdown + +
+ + +
+ +
+

πŸ“š Tested / Compatible

+ https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm + https://arubanetworking.hpe.com/techdocs/AOS_8.x_RN_WebHelp/Content/all-releases.htm +
+ +
+
+ + + + diff --git a/server.py b/server.py index 1942584..6c07e10 100644 --- a/server.py +++ b/server.py @@ -313,6 +313,43 @@ class Handler(BaseHTTPRequestHandler): self.wfile.write(json.dumps(data).encode()) def do_GET(self): + # Static file serving: or /url2pdf-tools/ β†’ /opt/url2pdf/ + # (nginx strips /url2pdf prefix; we accept both with and without it) + # Static file serving: or /url2pdf-tools/ -> /opt/url2pdf/ + # (nginx strips /url2pdf prefix; we accept both with and without it) + if self.path.startswith("/url2pdf-tools/"): + rel = self.path[len("/url2pdf-tools/"):] + elif "." in self.path.split("?")[0].split("/")[-1]: + rel = self.path.lstrip("/") + else: + rel = None + # Always serve index.html for root paths or empty + if rel is None or rel == "" or rel == "/": + rel = "index.html" + # Prevent path traversal + safe = os.path.normpath(rel).lstrip("/") + if safe.startswith("..") or "/.." in safe: + self.send_error(400, "Bad path") + return + fp = os.path.join("/opt/url2pdf", safe) + fp = os.path.join("/opt/url2pdf", safe) + if os.path.isfile(fp): + import mimetypes + ct, _ = mimetypes.guess_type(fp) + if ct is None: + ct = "application/octet-stream" + with open(fp, "rb") as f: + data = f.read() + self.send_response(200) + self.send_header("Content-Type", ct) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + # file not found under /opt/url2pdf - fall through to API or HTML fallback + self.send_error(404, "Not found: " + fp) + return + if self.path.startswith("/crawl/status"): qs = urllib.parse.urlparse(self.path).query params = urllib.parse.parse_qs(qs) @@ -327,6 +364,7 @@ class Handler(BaseHTTPRequestHandler): else: self._json(200, {"status": "pending"}) return + if self.path.startswith("/crawl/download"): qs = urllib.parse.urlparse(self.path).query @@ -424,30 +462,42 @@ class Handler(BaseHTTPRequestHandler): jdir = os.path.join(JOBS_DIR, job) os.makedirs(jdir, exist_ok=True) - # Write crawler script to npm dir so require() works - crawl_js = os.path.join(CWD, f"crawler_{job}.js") - with open(crawl_js, "w") as f: - f.write(CRAWLER_JS) + # Use v2 crawler (two-phase: discover versions + sub-pages) + v2_crawler = os.path.join(CWD, "crawl_hpe_v2.js") + if not os.path.exists(v2_crawler): + self._json(500, {"error": "v2 crawler not installed at /opt/url2pdf/crawl_hpe_v2.js"}) + return # Write initial status with open(os.path.join(jdir, "status.json"), "w") as f: json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f) - # Start background crawler + # Start background crawler (v2 writes output.md to /tmp/hpe-aruba-aos10-rn/, so copy to jdir) def run(): - subprocess.run( - ["timeout", "600", "node", crawl_js, url, jdir], + r = subprocess.run( + ["timeout", "900", "node", v2_crawler, url], capture_output=True, text=True, cwd=CWD, env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")} ) - # Clean up script - try: os.unlink(crawl_js) - except: pass + # Copy v2 output to jdir + import shutil + src_md = "/tmp/hpe-aruba-aos10-rn/all-releases-v2.md" + dst_md = os.path.join(jdir, "output.md") + if os.path.exists(src_md): + shutil.copy(src_md, dst_md) + # Update status + with open(os.path.join(jdir, "status.json"), "w") as f: + json.dump({ + "status": "done" if r.returncode == 0 else f"failed:{r.returncode}", + "time": time.strftime("%Y-%m-%dT%H:%M:%S"), + "output_size": os.path.getsize(dst_md) if os.path.exists(dst_md) else 0, + "log_tail": r.stdout[-500:] if r.stdout else r.stderr[-500:] + }, f) t = threading.Thread(target=run, daemon=True) t.start() - self._json(200, {"job": job, "status": "started"}) + self._json(200, {"job": job, "status": "started", "crawler": "v2"}) else: self.send_error(404)