// Crawl HPE Aruba Networking AOS 10.x Release Notes - v2
// Phase 1: discover all version overview pages
// Phase 2: for each version, fetch features/resolved/known sub-pages
const puppeteer = require('puppeteer-core');
const fs = require('fs');

// Auto-detect Chrome across Mac / Linux / Windows
function findChrome() {
  const fs = require('fs');
  const path = require('path');
  const homedir = require('os').homedir();
  const candidates = [
    // macOS (local dev)
    path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.97/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
    path.join(homedir, '.cache/puppeteer/chrome/mac_arm-148.0.7778.167/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing'),
    // Linux
    '/usr/bin/google-chrome',
    '/usr/bin/chromium-browser',
    '/usr/bin/chromium',
    // Docker puppeteer cache
    path.join(homedir, '.cache/puppeteer/chrome/'),
  ];
  for (const c of candidates) {
    if (fs.existsSync(c)) return c;
  }
  return null;
}
const CHROME_PATH = findChrome();
if (!CHROME_PATH) {
  console.error('No Chrome found. Try: apt install chromium / google-chrome-stable');
  process.exit(1);
}
console.log('Using Chrome:', CHROME_PATH);
const BASE = process.argv[2] || 'https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm';
const OUT_DIR = '/tmp/hpe-aruba-aos10-rn';
const VERSIONS_FILE = OUT_DIR + '/versions.json';
const MD_FILE = OUT_DIR + '/all-releases-v2.md';

if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });

const CHROME_ARGS = ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'];
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
const EXTRA_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
  'Accept-Language': 'en-US,en;q=0.9',
  'Accept-Encoding': 'gzip, deflate, br'
};

const SUB_PAGE_PATTERNS = [
  { regex: /features[/-]\d+\.htm/i, type: 'features' },
  { regex: /resolved-issues[/-]\d+\.htm/i, type: 'resolved' },
  { regex: /known-issues[/-]\d+\.htm/i, type: 'known' },
  { regex: /regulatory[/-]\d+\.htm/i, type: 'regulatory' },
];

async function setupPage(browser) {
  const page = await browser.newPage();
  await page.setUserAgent(USER_AGENT);
  await page.setExtraHTTPHeaders(EXTRA_HEADERS);
  // Skip script/style in extraction
  await page.evaluateOnNewDocument(() => {
    window.MutationObserver = window.MutationObserver || undefined;
  });
  return page;
}

// Clean DOM extraction: skip script/style blocks
async function extractPage(page, href) {
  await page.goto(href, { waitUntil: 'networkidle2', timeout: 30000 });
  await new Promise(r => setTimeout(r, 600));

  return await page.evaluate(() => {
    // Remove script + style + nav elements from clone
    const document2 = document.cloneNode(true);
    document2.querySelectorAll('script, style, nav, .MCWebHelpFramesetLink, .toolbar, .navigation').forEach(el => el.remove());

    const mainSel = ['article', '.content', '.main-content', '.doc-content', 'main', '#content', '.topic-content', '.body-container'];
    let main = document2.body;
    for (const sel of mainSel) {
      const el = document2.querySelector(sel);
      if (el) { main = el; break; }
    }

    const h1 = main.querySelector('h1, h2');
    const title = h1 ? h1.textContent.trim() : document2.title.trim();

    const blocks = [];
    main.querySelectorAll('h1, h2, h3, h4, p, li, pre, code, table').forEach(el => {
      const txt = el.textContent.trim();
      if (!txt || txt.length < 3) return;
      const tag = el.tagName;
      if (tag.match(/^H[1-4]$/)) {
        blocks.push('\n' + '#'.repeat(parseInt(tag[1]) + 1) + ' ' + txt + '\n');
      } else if (tag === 'LI') {
        blocks.push('- ' + txt);
      } else if (tag === 'PRE' || tag === 'CODE') {
        blocks.push('\n```\n' + txt + '\n```\n');
      } else if (tag === 'TABLE') {
        const rows = [];
        el.querySelectorAll('tr').forEach(tr => {
          const cells = Array.from(tr.querySelectorAll('th, td')).map(c => c.textContent.trim()).join(' | ');
          if (cells) rows.push('| ' + cells + ' |');
        });
        if (rows.length) blocks.push('\n' + rows.join('\n') + '\n');
      } else {
        blocks.push(txt);
      }
    });
    return { title, body: blocks.join('\n\n') };
  });
}

async function discoverVersions(page) {
  if (fs.existsSync(VERSIONS_FILE)) {
    const versions = JSON.parse(fs.readFileSync(VERSIONS_FILE, 'utf8'));
    console.log(`📋 ${versions.length} versions (cached)`);
    return versions;
  }
  await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 60000 });
  await new Promise(r => setTimeout(r, 3000));

  const baseUrl = new URL(BASE);
  const articleDir = baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1);

  const allLinks = await page.evaluate(() => {
    const links = [];
    document.querySelectorAll('a[href]').forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 200) });
    });
    return links;
  });

  const seen = new Set();
  const versions = [];
  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (!u.pathname.startsWith(articleDir)) continue;
      if (u.hash) continue;
      if (/javascript:|^#|^\?/i.test(link.href)) continue;
      const fn = u.pathname.split('/').pop();
      if (!fn || fn.length < 3) continue;
      if (!seen.has(fn)) {
        seen.add(fn);
        versions.push({ href: link.href, text: link.text });
      }
    } catch (e) {}
  }

  fs.writeFileSync(VERSIONS_FILE, JSON.stringify(versions, null, 2));
  console.log(`📋 ${versions.length} version pages found`);
  return versions;
}

async function discoverSubPages(page, versionUrl) {
  await page.goto(versionUrl, { waitUntil: 'networkidle2', timeout: 30000 });
  await new Promise(r => setTimeout(r, 1500));
  return await page.evaluate(() => {
    const links = [];
    document.querySelectorAll('a[href]').forEach(a => {
      const href = a.href;
      const text = a.textContent.trim();
      if (/features|resolved-issues|known-issues|regulatory|whats-new/.test(href) && !/^#/.test(href) && !/Send Feedback/.test(text)) {
        links.push({ href, text: text.slice(0, 80) });
      }
    });
    return links;
  });
}

async function crawl() {
  const browser = await puppeteer.launch({
    executablePath: CHROME_PATH,
    headless: true,
    args: CHROME_ARGS
  });
  const page = await setupPage(browser);

  const versions = await discoverVersions(page);
  console.log(`\n📥 Phase 1 (${versions.length} version overviews)...\n`);

  fs.writeFileSync(MD_FILE, `# HPE Aruba Networking — AOS 10.x Release Notes (v2)\n\n> Source: ${BASE}\n> Versions: ${versions.length}\n> Crawled: ${new Date().toISOString()}\n\n---\n\n`);

  let ok = 0, fail = 0;
  const allSubPages = [];

  for (let i = 0; i < versions.length; i++) {
    const { href, text } = versions[i];
    try {
      const content = await extractPage(page, href);
      const md = `## ${i + 1}. ${content.title}\n\n_Source: ${href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      ok++;
      console.log(`✅ [${i+1}/${versions.length}] ${content.title.slice(0, 50).replace(/\n/g, ' ')}`);

      // Phase 2: discover sub-pages for this version
      const subPages = await discoverSubPages(page, href);
      for (const sub of subPages) {
        // Dedup: only add sub-pages for this specific version
        if (sub.href.includes(href.split('/').slice(0, -1).join('/')) || sub.href.includes('/10.')) {
          allSubPages.push({ parent: text, parentHref: href, ...sub });
        }
      }
    } catch (e) {
      fail++;
      const md = `## ${i + 1}. ${text}\n\n_❌ Error: ${e.message}_\n\n_Source: ${href}_\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      console.log(`❌ [${i+1}/${versions.length}] ${e.message.slice(0, 80)}`);
    }
  }

  // Dedupe sub-pages
  const seenSub = new Set();
  const uniqueSub = allSubPages.filter(s => {
    if (seenSub.has(s.href)) return false;
    seenSub.add(s.href);
    return true;
  });

  console.log(`\n📥 Phase 2 (${uniqueSub.length} sub-pages: features/resolved/known)...\n`);

  fs.appendFileSync(MD_FILE, `\n\n# === Detail Sub-Pages (Features / Resolved Issues / Known Issues) ===\n\n`);

  let subOk = 0, subFail = 0;
  for (let i = 0; i < uniqueSub.length; i++) {
    const sub = uniqueSub[i];
    const type = SUB_PAGE_PATTERNS.find(p => p.regex.test(sub.href))?.type || 'detail';
    try {
      const content = await extractPage(page, sub.href);
      const md = `## ${sub.parent} — ${type.toUpperCase()}\n\n_${sub.text}_\n\n_Source: ${sub.href}_\n\n${content.body || '_no content_'}\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      subOk++;
      console.log(`✅ [${i+1}/${uniqueSub.length}] ${type.padEnd(10)} ${sub.parent} — ${sub.text.slice(0, 40)}`);
    } catch (e) {
      subFail++;
      const md = `## ${sub.parent} — ${type}\n\n_❌ Error: ${e.message}_\n\n_Source: ${sub.href}_\n\n---\n\n`;
      fs.appendFileSync(MD_FILE, md);
      console.log(`❌ [${i+1}/${uniqueSub.length}] ${type} ${e.message.slice(0, 60)}`);
    }
  }

  await browser.close();
  console.log(`\n📊 Done:`);
  console.log(`   Phase 1 (overviews): ${ok} OK, ${fail} failed`);
  console.log(`   Phase 2 (sub-pages): ${subOk} OK, ${subFail} failed`);
  console.log(`📁 Output: ${MD_FILE}`);
}

crawl().catch(err => { console.error(err); process.exit(1); });