url2pdf-crawler-local/crawl_md.js

// Crawl Trend Micro docs and convert to a single Markdown file
const puppeteer = require("puppeteer");
const fs = require("fs");

const BASE = process.argv[2];
const OUT = "/tmp/apex_one_admin_guide.md";
const baseUrl = new URL(BASE);
const articleIdx = baseUrl.pathname.lastIndexOf("/");
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);

(async () => {
  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
  const mainPage = await browser.newPage();

  console.log("🌐 Loading TOC:", BASE);
  await mainPage.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
  await new Promise(r => setTimeout(r, 3000));

  // Extract all doc links
  const allLinks = await mainPage.evaluate(() => {
    const links = [];
    document.querySelectorAll("a[href]").forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
    });
    return links;
  });
  await mainPage.close();

  // Filter: same article dir, en-us only, unique filenames (same logic as PDF crawler)
  const seenFiles = new Set();
  const docPages = [];
  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (!u.pathname.startsWith(articleDir)) continue;
      if (!u.pathname.startsWith("/en-us/")) continue;
      if (u.hash) continue;
      const fn = u.pathname.split("/").pop();
      if (!fn || fn.length < 20) continue;
      const lower = link.text.toLowerCase();
      if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
      if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
      if (!seenFiles.has(fn)) {
        seenFiles.add(fn);
        docPages.push({ href: link.href, text: link.text });
      }
    } catch(e) {}
  }

  console.log(`📋 ${docPages.length} pages to extract`);

  const stream = fs.createWriteStream(OUT);
  stream.write("# Apex One Service Pack 1 (2025) — Admin Guide\n\n");
  stream.write(`> Crawled from ${BASE}\n\n`);
  stream.write("---\n\n");

  for (let i = 0; i < docPages.length; i++) {
    const { href, text } = docPages[i];
    try {
      const page = await browser.newPage();
      await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
      await new Promise(r => setTimeout(r, 500));

      // Extract title + body content
      const content = await page.evaluate(() => {
        // Try to find the main content area
        const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content", ".article-content"];
        let main = null;
        for (const sel of mainSel) {
          main = document.querySelector(sel);
          if (main) break;
        }
        if (!main) main = document.body;

        // Get H1 first
        const h1 = main.querySelector("h1, h2");
        const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim();

        // Get all text content, preserving some structure
        const blocks = [];
        main.querySelectorAll("h1, h2, h3, h4, p, li, table, pre, .note, .warning, .tip").forEach(el => {
          const tag = el.tagName;
          const txt = el.textContent.trim();
          if (!txt) return;
          if (tag.match(/^H[1-4]$/)) {
            const level = parseInt(tag[1]);
            blocks.push("\n" + "#".repeat(level + 1) + " " + txt + "\n");
          } else if (tag === "LI") {
            blocks.push("- " + txt);
          } else if (tag === "PRE") {
            blocks.push("\n```\n" + txt + "\n```\n");
          } else if (tag === "TABLE") {
            blocks.push("\n<!-- table omitted -->\n");
          } else {
            blocks.push(txt);
          }
        });

        return { title, body: blocks.join("\n\n") };
      });

      await page.close();

      stream.write(`## ${i + 1}. ${content.title}\n\n`);
      stream.write(content.body || "_No content extracted_\n");
      stream.write("\n---\n\n");

      if ((i + 1) % 50 === 0) console.log(`  ${i + 1}/${docPages.length}`);
    } catch(e) {
      console.log(`  ⚠️ [${i+1}] ${text.slice(0,40)}: ${e.message.slice(0,50)}`);
      stream.write(`## ${i + 1}. ${text}\n\n_Error: ${e.message}_\n\n---\n\n`);
    }
  }

  stream.end();
  await browser.close();

  const stats = fs.statSync(OUT);
  console.log(`\n✅ ${OUT} (${(stats.size/1024).toFixed(0)}KB)`);
})();