v2: two-phase doc crawler + web UI + server.py static file serving

- crawl_hpe_v2.js: two-phase (discover versions + sub-pages: features/resolved/known/regulatory)
- crawl_hpe.js: v1 single-phase
- hpe-aos10-versions.json: 60 versions cache
- server.py:
  - /url2pdf-tools/ now serves index.html (web UI)
  - /url2pdf-tools/<file> serves from /opt/url2pdf/ (with path traversal protection)
  - /url2pdf/crawl uses v2 crawler (was inline CRAWLER_JS hardcoded for Trend Micro)
- index.html: new web UI (paste URL → crawl → download)

server.py

@@ -313,6 +313,43 @@ class Handler(BaseHTTPRequestHandler):
         self.wfile.write(json.dumps(data).encode())
     
     def do_GET(self):
+        # Static file serving: <file> or /url2pdf-tools/<file> → /opt/url2pdf/<file>
+        # (nginx strips /url2pdf prefix; we accept both with and without it)
+        # Static file serving: <file> or /url2pdf-tools/<file> -> /opt/url2pdf/<file>
+        # (nginx strips /url2pdf prefix; we accept both with and without it)
+        if self.path.startswith("/url2pdf-tools/"):
+            rel = self.path[len("/url2pdf-tools/"):]
+        elif "." in self.path.split("?")[0].split("/")[-1]:
+            rel = self.path.lstrip("/")
+        else:
+            rel = None
+        # Always serve index.html for root paths or empty
+        if rel is None or rel == "" or rel == "/":
+            rel = "index.html"
+        # Prevent path traversal
+        safe = os.path.normpath(rel).lstrip("/")
+        if safe.startswith("..") or "/.." in safe:
+            self.send_error(400, "Bad path")
+            return
+        fp = os.path.join("/opt/url2pdf", safe)
+        fp = os.path.join("/opt/url2pdf", safe)
+        if os.path.isfile(fp):
+            import mimetypes
+            ct, _ = mimetypes.guess_type(fp)
+            if ct is None:
+                ct = "application/octet-stream"
+            with open(fp, "rb") as f:
+                data = f.read()
+            self.send_response(200)
+            self.send_header("Content-Type", ct)
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+        # file not found under /opt/url2pdf - fall through to API or HTML fallback
+        self.send_error(404, "Not found: " + fp)
+        return
+
         if self.path.startswith("/crawl/status"):
             qs = urllib.parse.urlparse(self.path).query
             params = urllib.parse.parse_qs(qs)
@@ -327,6 +364,7 @@ class Handler(BaseHTTPRequestHandler):
             else:
                 self._json(200, {"status": "pending"})
             return
+
         
         if self.path.startswith("/crawl/download"):
             qs = urllib.parse.urlparse(self.path).query
@@ -424,30 +462,42 @@ class Handler(BaseHTTPRequestHandler):
             jdir = os.path.join(JOBS_DIR, job)
             os.makedirs(jdir, exist_ok=True)
             
-            # Write crawler script to npm dir so require() works
-            crawl_js = os.path.join(CWD, f"crawler_{job}.js")
-            with open(crawl_js, "w") as f:
-                f.write(CRAWLER_JS)
+            # Use v2 crawler (two-phase: discover versions + sub-pages)
+            v2_crawler = os.path.join(CWD, "crawl_hpe_v2.js")
+            if not os.path.exists(v2_crawler):
+                self._json(500, {"error": "v2 crawler not installed at /opt/url2pdf/crawl_hpe_v2.js"})
+                return
             
             # Write initial status
             with open(os.path.join(jdir, "status.json"), "w") as f:
                 json.dump({"status": "starting", "time": time.strftime("%Y-%m-%dT%H:%M:%S")}, f)
             
-            # Start background crawler
+            # Start background crawler (v2 writes output.md to /tmp/hpe-aruba-aos10-rn/, so copy to jdir)
             def run():
-                subprocess.run(
-                    ["timeout", "600", "node", crawl_js, url, jdir],
+                r = subprocess.run(
+                    ["timeout", "900", "node", v2_crawler, url],
                     capture_output=True, text=True, cwd=CWD,
                     env={**os.environ, "NODE_PATH": os.path.join(CWD, "node_modules")}
                 )
-                # Clean up script
-                try: os.unlink(crawl_js)
-                except: pass
+                # Copy v2 output to jdir
+                import shutil
+                src_md = "/tmp/hpe-aruba-aos10-rn/all-releases-v2.md"
+                dst_md = os.path.join(jdir, "output.md")
+                if os.path.exists(src_md):
+                    shutil.copy(src_md, dst_md)
+                # Update status
+                with open(os.path.join(jdir, "status.json"), "w") as f:
+                    json.dump({
+                        "status": "done" if r.returncode == 0 else f"failed:{r.returncode}",
+                        "time": time.strftime("%Y-%m-%dT%H:%M:%S"),
+                        "output_size": os.path.getsize(dst_md) if os.path.exists(dst_md) else 0,
+                        "log_tail": r.stdout[-500:] if r.stdout else r.stderr[-500:]
+                    }, f)
             
             t = threading.Thread(target=run, daemon=True)
             t.start()
             
-            self._json(200, {"job": job, "status": "started"})
+            self._json(200, {"job": job, "status": "started", "crawler": "v2"})
         
         else:
             self.send_error(404)