url2pdf-crawler-local/crawl_md_batch.js

// Batch crawl: process 100 pages, close/reopen browser to free memory
const puppeteer = require("puppeteer");
const fs = require("fs");

const BASE = process.argv[2];
const START = parseInt(process.argv[3]) || 0;
const BATCH = parseInt(process.argv[4]) || 100;
const OUT = "/tmp/apex_one_admin_guide.md";
const URLS_FILE = "/tmp/apex_one_urls.json";
const baseUrl = new URL(BASE);
const articleIdx = baseUrl.pathname.lastIndexOf("/");
const articleDir = baseUrl.pathname.substring(0, articleIdx + 1);

async function getUrls() {
  if (fs.existsSync(URLS_FILE)) {
    return JSON.parse(fs.readFileSync(URLS_FILE, "utf8"));
  }
  
  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
  const page = await browser.newPage();
  await page.goto(BASE, { waitUntil: "networkidle2", timeout: 30000 });
  await new Promise(r => setTimeout(r, 3000));

  const allLinks = await page.evaluate(() => {
    const links = [];
    document.querySelectorAll("a[href]").forEach(a => {
      const text = a.textContent.trim();
      if (text.length > 2) links.push({ href: a.href, text: text.slice(0, 80) });
    });
    return links;
  });
  await browser.close();

  const seenFiles = new Set();
  const docPages = [];
  for (const link of allLinks) {
    try {
      const u = new URL(link.href);
      if (u.hostname !== baseUrl.hostname) continue;
      if (!u.pathname.startsWith(articleDir)) continue;
      if (!u.pathname.startsWith("/en-us/")) continue;
      if (u.hash) continue;
      const fn = u.pathname.split("/").pop();
      if (!fn || fn.length < 20) continue;
      const lower = link.text.toLowerCase();
      if (/english|dansk|deutsch|español|français|italiano|nederlands|norsk|polski|português|svenska|ภาษาไทย|tiếng việt|türkçe|čeština|ελληνικά|български|русский|עברית|العربية|日本語|简体中文|繁體中文|한국어|bahasa indonesia/i.test(lower)) continue;
      if (/online help center|security for endpoints|apex one$/i.test(lower)) continue;
      if (!seenFiles.has(fn)) {
        seenFiles.add(fn);
        docPages.push({ href: link.href, text: link.text });
      }
    } catch(e) {}
  }
  fs.writeFileSync(URLS_FILE, JSON.stringify(docPages, null, 2));
  return docPages;
}

async function extractBatch(urls, start, count) {
  const browser = await puppeteer.launch({ headless: true, args: ["--no-sandbox"] });
  const end = Math.min(start + count, urls.length);
  
  for (let i = start; i < end; i++) {
    const { href, text } = urls[i];
    try {
      const page = await browser.newPage();
      await page.goto(href, { waitUntil: "networkidle2", timeout: 20000 });
      await new Promise(r => setTimeout(r, 300));

      const content = await page.evaluate(() => {
        const mainSel = ["article", ".content", ".main-content", ".doc-content", "main", "#content"];
        let main = document.body;
        for (const sel of mainSel) {
          const el = document.querySelector(sel);
          if (el) { main = el; break; }
        }
        const h1 = main.querySelector("h1, h2");
        const title = h1 ? h1.textContent.trim() : document.title.replace(" | TrendAI™", "").trim();
        
        const blocks = [];
        main.querySelectorAll("h1, h2, h3, h4, p, li, pre").forEach(el => {
          const txt = el.textContent.trim();
          if (!txt || txt.length < 3) return;
          const tag = el.tagName;
          if (tag.match(/^H[1-4]$/)) {
            blocks.push("\n" + "#".repeat(parseInt(tag[1]) + 1) + " " + txt + "\n");
          } else if (tag === "LI") {
            blocks.push("- " + txt);
          } else if (tag === "PRE") {
            blocks.push("\n```\n" + txt + "\n```\n");
          } else {
            blocks.push(txt);
          }
        });
        return { title, body: blocks.join("\n\n") };
      });

      await page.close();

      const out = `## ${i + 1}. ${content.title}\n\n${content.body || "_no content_"}  \n`;
      fs.appendFileSync(OUT, out);
    } catch(e) {
      fs.appendFileSync(OUT, `## ${i + 1}. ${text}\n\n_Error_  \n`);
    }
  }
  
  await browser.close();
  console.log(`✅ [${start+1}-${end}/${urls.length}]`);
}

(async () => {
  const urls = await getUrls();
  console.log(`📋 ${urls.length} pages total, starting at ${START}, batch ${BATCH}`);
  
  // Write header if starting from 0
  if (START === 0) {
    fs.writeFileSync(OUT, `# Apex One Service Pack 1 (2025) — Admin Guide\n\n> ${urls.length} pages from ${BASE}\n\n`);
  }
  
  await extractBatch(urls, START, BATCH);
})();