IT狗
e6116ecc7d
- crawl_hpe_v2.js: two-phase (discover versions + sub-pages: features/resolved/known/regulatory) - crawl_hpe.js: v1 single-phase - hpe-aos10-versions.json: 60 versions cache - server.py: - /url2pdf-tools/ now serves index.html (web UI) - /url2pdf-tools/<file> serves from /opt/url2pdf/ (with path traversal protection) - /url2pdf/crawl uses v2 crawler (was inline CRAWLER_JS hardcoded for Trend Micro) - index.html: new web UI (paste URL → crawl → download)
143 lines
6.8 KiB
HTML
143 lines
6.8 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>URL-to-PDF · Doc Crawler</title>
|
|
<style>
|
|
*{box-sizing:border-box;margin:0;padding:0}
|
|
body{font-family:-apple-system,system-ui,sans-serif;background:#0a0a0f;color:#f0f0f4;min-height:100vh;display:flex;align-items:center;justify-content:center;padding:20px}
|
|
.card{background:#131317;border:1px solid #2a2a33;border-radius:16px;padding:32px;max-width:720px;width:100%}
|
|
h1{font-size:1.5rem;font-weight:700;margin-bottom:6px;letter-spacing:-0.3px}
|
|
.sub{color:#88889a;font-size:0.85rem;margin-bottom:24px}
|
|
.tag{display:inline-block;padding:3px 10px;border-radius:6px;background:rgba(139,92,246,0.15);color:#a78bfa;font-size:0.7rem;font-weight:600;margin-right:6px;margin-bottom:8px}
|
|
.row{display:flex;gap:8px;margin-bottom:12px}
|
|
input{flex:1;padding:12px 16px;border-radius:10px;border:1px solid #2a2a33;background:#0a0a0f;color:#f0f0f4;font-size:0.9rem;font-family:inherit;transition:.2s}
|
|
input:focus{outline:none;border-color:#8b5cf6;box-shadow:0 0 0 3px rgba(139,92,246,0.1)}
|
|
button{padding:12px 20px;border-radius:10px;border:none;background:#8b5cf6;color:#fff;font-size:0.9rem;font-weight:600;cursor:pointer;font-family:inherit;transition:.2s;white-space:nowrap}
|
|
button:hover{background:#7c3aed}
|
|
button:disabled{opacity:0.4;cursor:not-allowed}
|
|
.examples{background:#0a0a0f;border:1px solid #2a2a33;border-radius:10px;padding:12px;margin-bottom:16px;font-size:0.8rem}
|
|
.examples h3{font-size:0.75rem;color:#88889a;margin-bottom:8px;text-transform:uppercase;letter-spacing:0.5px}
|
|
.examples a{color:#a78bfa;text-decoration:none;display:block;padding:4px 0;font-family:monospace;font-size:0.75rem}
|
|
.examples a:hover{color:#8b5cf6}
|
|
.status{margin-top:16px;padding:14px 16px;border-radius:10px;font-size:0.85rem;display:none}
|
|
.status.active{display:block}
|
|
.status.info{background:rgba(96,165,250,0.1);color:#60a5fa;border:1px solid rgba(96,165,250,0.3)}
|
|
.status.success{background:rgba(16,185,129,0.1);color:#34d399;border:1px solid rgba(16,185,129,0.3)}
|
|
.status.error{background:rgba(239,68,68,0.1);color:#f87171;border:1px solid rgba(239,68,68,0.3)}
|
|
.progress{height:4px;background:#1a1a22;border-radius:2px;margin-top:8px;overflow:hidden}
|
|
.progress-bar{height:100%;background:#8b5cf6;border-radius:2px;transition:width .3s;width:0}
|
|
.result{margin-top:12px;font-size:0.8rem}
|
|
.result a{color:#34d399;text-decoration:none;font-weight:600}
|
|
.result a:hover{text-decoration:underline}
|
|
.info-box{background:#0a0a0f;border:1px solid #2a2a33;border-radius:10px;padding:14px;margin-bottom:16px;font-size:0.78rem;color:#bcbcc8;line-height:1.6}
|
|
.info-box code{background:#1a1a22;padding:2px 6px;border-radius:4px;font-size:0.72rem;color:#a78bfa}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="card">
|
|
<h1>📄 URL-to-PDF · Doc Crawler</h1>
|
|
<p class="sub">Paste a documentation URL → get all sub-pages as Markdown</p>
|
|
|
|
<div class="info-box">
|
|
🐕 <strong>IT狗 2026-07-24</strong> — Two-phase crawler: discovers version pages, then fetches each version's <em>features / resolved / known issues / regulatory</em> sub-pages. Currently tested with HPE AOS 10.x (209 pages).
|
|
</div>
|
|
|
|
<span class="tag">v2</span><span class="tag">Two-phase</span><span class="tag">Markdown</span>
|
|
|
|
<div class="row">
|
|
<input id="urlInput" type="url" placeholder="https://example.com/docs/all-releases.htm" value="https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm">
|
|
<button id="goBtn" onclick="startCrawl()">Crawl</button>
|
|
</div>
|
|
|
|
<div class="examples">
|
|
<h3>📚 Tested / Compatible</h3>
|
|
<a href="#" onclick="event.preventDefault();document.getElementById(\"urlInput\").value=this.textContent">https://arubanetworking.hpe.com/techdocs/AOS_10.x_RN_WebHelp/Content/all-releases.htm</a>
|
|
<a href="#" onclick="event.preventDefault();document.getElementById(\"urlInput\").value=this.textContent">https://arubanetworking.hpe.com/techdocs/AOS_8.x_RN_WebHelp/Content/all-releases.htm</a>
|
|
</div>
|
|
|
|
<div id="status" class="status"></div>
|
|
</div>
|
|
|
|
<script>
|
|
const API = "/url2pdf";
|
|
const input = document.getElementById("urlInput");
|
|
const btn = document.getElementById("goBtn");
|
|
const status = document.getElementById("status");
|
|
|
|
async function startCrawl() {
|
|
const url = (input.value || "").trim();
|
|
if (!url) { showStatus("error", "Please enter a URL"); return; }
|
|
if (!/^https?:\/\//.test(url)) { showStatus("error", "URL must start with http:// or https://"); return; }
|
|
|
|
btn.disabled = true;
|
|
showStatus("info", "🚀 Starting crawl...");
|
|
|
|
try {
|
|
const r = await fetch(API + "/crawl", {
|
|
method: "POST",
|
|
headers: {"Content-Type": "application/json"},
|
|
body: JSON.stringify({url})
|
|
});
|
|
const data = await r.json();
|
|
if (data.error) throw new Error(data.error);
|
|
|
|
const job = data.job;
|
|
showStatus("info", `⏳ Crawl started (job: <code>${job}</code>). Polling status...`);
|
|
|
|
// Poll status
|
|
const startTime = Date.now();
|
|
let pollCount = 0;
|
|
while (true) {
|
|
await new Promise(r => setTimeout(r, 5000));
|
|
pollCount++;
|
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
|
|
|
const sr = await fetch(API + "/crawl/status?job=" + job);
|
|
const sd = await sr.json();
|
|
|
|
const statusText = sd.status || "pending";
|
|
const sizeMB = sd.output_size ? (sd.output_size / 1024 / 1024).toFixed(1) + " MB" : "";
|
|
|
|
if (statusText === "done") {
|
|
showStatus("success", `✅ Done in ${elapsed}s — ${sizeMB} of markdown ready`, job);
|
|
btn.disabled = false;
|
|
return;
|
|
} else if (statusText.startsWith("failed")) {
|
|
showStatus("error", `❌ Failed: ${statusText}<br><pre style="white-space:pre-wrap;font-size:0.7rem;color:#f87171;margin-top:8px;max-height:200px;overflow:auto">${sd.log_tail || ""}</pre>`);
|
|
btn.disabled = false;
|
|
return;
|
|
} else if (statusText.startsWith("done:")) {
|
|
// legacy format
|
|
showStatus("success", `✅ ${statusText} (${elapsed}s)`, job);
|
|
btn.disabled = false;
|
|
return;
|
|
} else {
|
|
showStatus("info", `⏳ ${statusText} (${elapsed}s elapsed, poll #${pollCount})`);
|
|
}
|
|
|
|
if (elapsed > 1200) { // 20min timeout
|
|
showStatus("error", "⏱️ Timeout (>20min). Check job manually.");
|
|
btn.disabled = false;
|
|
return;
|
|
}
|
|
}
|
|
} catch (e) {
|
|
showStatus("error", "❌ Error: " + e.message);
|
|
btn.disabled = false;
|
|
}
|
|
}
|
|
|
|
function showStatus(type, html, job) {
|
|
status.className = "status active " + type;
|
|
const jobLink = job ? `<div class="result">📥 <a href="${API}/crawl/download?job=${job}&format=md" download>Download output.md</a></div>` : "";
|
|
status.innerHTML = html + jobLink;
|
|
}
|
|
|
|
// Allow Enter key
|
|
input.addEventListener("keypress", e => { if (e.key === "Enter") startCrawl(); });
|
|
</script>
|
|
</body>
|
|
</html>
|