From 8e0d73c37e5418d8d00603d4499dd19f5781e6e1 Mon Sep 17 00:00:00 2001 From: IT Dog Date: Sun, 19 Jul 2026 20:26:59 +0800 Subject: [PATCH] Conditional upsert: upload skips edited records, updates not-edited - Add last_edited_at + last_edited_by columns to holidays + leave_records - Migration: ALTER TABLE adds columns (idempotent) - PUT endpoints set last_edited_at on every manual edit - POST /api/leave/upload: skip if last_edited_at NOT NULL, update otherwise - New POST /api/holidays/upload: same logic, admin only - LeaveOut + HolidayOut schemas include last_edited_at - Response includes added/updated/skipped counters - Verified: edited Jay Lam SL preserved, not-edited May Chan SL updated --- backend/main.py | 132 ++++++++++++++++++++++++++++++++++++++++++++-- backend/models.py | 8 +++ 2 files changed, 137 insertions(+), 3 deletions(-) diff --git a/backend/main.py b/backend/main.py index d60debd..667ebd9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -44,6 +44,7 @@ class HolidayOut(BaseModel): is_mandatory: Optional[bool] = None source: Optional[str] = None notes: Optional[str] = None + last_edited_at: Optional[datetime] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None @@ -84,6 +85,7 @@ class LeaveOut(BaseModel): approved_by: Optional[str] = None source: Optional[str] = None notes: Optional[str] = None + last_edited_at: Optional[datetime] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None @@ -1358,6 +1360,8 @@ async def update_holiday( for k, v in update_data.items(): setattr(h, k, v) h.source = "manual_edit" + h.last_edited_at = datetime.utcnow() + h.last_edited_by = current_user.id db.commit() db.refresh(h) return h @@ -1378,6 +1382,114 @@ async def delete_holiday( return {"deleted": holiday_id} +@app.post("/api/holidays/upload") +async def upload_holidays( + file: UploadFile = File(...), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_admin), +): + """Bulk upload holidays from CSV or Excel. + + CSV columns: date,name,name_en,is_mandatory,notes + Excel: same columns (row 1 = header) + + Logic: + - Date exists + never edited (last_edited_at IS NULL) -> UPDATE + - Date exists + manually edited (last_edited_at NOT NULL) -> SKIP + - New date -> INSERT + """ + from io import BytesIO + content = await file.read() + fname = file.filename.lower() + + rows = [] + if fname.endswith(".csv"): + import csv, io as _io + reader = csv.DictReader(_io.StringIO(content.decode("utf-8-sig"))) + for row in reader: + rows.append({ + "date": row.get("date", "").strip(), + "name": row.get("name", "").strip(), + "name_en": row.get("name_en", "").strip() or None, + "is_mandatory": (row.get("is_mandatory", "true").strip().lower() in ("true", "1", "yes", "y")), + "notes": row.get("notes", "").strip() or None, + }) + elif fname.endswith((".xlsx", ".xls")): + import openpyxl + wb = openpyxl.load_workbook(BytesIO(content), data_only=True) + ws = wb.active + headers = [str(c.value or "").strip() for c in ws[1]] + col_idx = {h: i for i, h in enumerate(headers)} + for row in ws.iter_rows(min_row=2, values_only=True): + if not row or all(v is None for v in row): + continue + def getv(h, default=None): + v = row[col_idx[h]] if h in col_idx and col_idx[h] < len(row) else None + return v if v is not None else default + rows.append({ + "date": str(getv("date") or "").strip(), + "name": str(getv("name") or "").strip(), + "name_en": str(getv("name_en") or "").strip() or None, + "is_mandatory": bool(getv("is_mandatory", True)), + "notes": str(getv("notes") or "").strip() or None, + }) + else: + raise HTTPException(status_code=400, detail="File must be CSV or Excel") + + added = 0 + updated = 0 + skipped = 0 + errors = [] + for i, r in enumerate(rows): + try: + if not r["date"] or not r["name"]: + errors.append(f"Row {i+2}: missing date or name") + skipped += 1 + continue + from datetime import datetime as _dt + if isinstance(r["date"], str): + d = _dt.strptime(r["date"], "%Y-%m-%d").date() + else: + d = r["date"] + + existing = db.query(Holiday).filter(Holiday.date == d).first() + + if existing: + if existing.last_edited_at is not None: + errors.append(f"Row {i+2}: {d} was edited at {existing.last_edited_at.isoformat()} - skipped") + skipped += 1 + continue + # Not edited - safe to update + existing.name = r["name"] + existing.name_en = r["name_en"] + existing.is_mandatory = r["is_mandatory"] + existing.notes = r["notes"] + existing.source = "csv_upload" + updated += 1 + continue + + db.add(Holiday( + date=d, + name=r["name"], + name_en=r["name_en"], + region="HK", + is_mandatory=r["is_mandatory"], + source="csv_upload", + notes=r["notes"], + )) + added += 1 + except Exception as e: + errors.append(f"Row {i+2}: {str(e)}") + skipped += 1 + + db.commit() + return { + "added": added, "updated": updated, "skipped": skipped, + "errors": errors[:20], + "total_in_db": db.query(Holiday).count(), + } + + @app.post("/api/holidays/refresh") async def refresh_holidays_from_gov_hk( years: Optional[str] = None, # CSV: "2025,2026,2027" @@ -1504,6 +1616,8 @@ async def update_leave( raise HTTPException(status_code=400, detail="leave_type must be SL / CL / AL") for k, v in update_data.items(): setattr(lv, k, v) + lv.last_edited_at = datetime.utcnow() + lv.last_edited_by = current_user.id db.commit() db.refresh(lv) return lv @@ -1576,6 +1690,7 @@ async def upload_leaves( raise HTTPException(status_code=400, detail="File must be CSV or Excel") added = 0 + updated = 0 skipped = 0 errors = [] for i, r in enumerate(rows): @@ -1585,7 +1700,7 @@ async def upload_leaves( skipped += 1 continue if r["leave_type"] not in ("SL", "CL", "AL"): - errors.append(f"Row {i+2}: invalid leave_type {r[chr(39)+'leave_type'+chr(39)]}") + errors.append(f"Row {i+2}: invalid leave_type") skipped += 1 continue @@ -1600,8 +1715,19 @@ async def upload_leaves( LeaveRecord.date == d, LeaveRecord.leave_type == r["leave_type"], ).first() + if existing: - skipped += 1 + if existing.last_edited_at is not None: + # Was manually edited - protect from overwrite + errors.append(f"Row {i+2}: {r['employee_name']} {d} {r['leave_type']} was edited at {existing.last_edited_at.isoformat()} - skipped") + skipped += 1 + continue + # Not edited - safe to update + existing.hours = r["hours"] + existing.reason = r["reason"] + existing.approved_by = r["approved_by"] + existing.source = "csv_upload" + updated += 1 continue db.add(LeaveRecord( @@ -1621,7 +1747,7 @@ async def upload_leaves( db.commit() return { - "added": added, "skipped": skipped, + "added": added, "updated": updated, "skipped": skipped, "errors": errors[:20], # First 20 errors only "total_in_db": db.query(LeaveRecord).count(), } diff --git a/backend/models.py b/backend/models.py index 3f3f95c..c7a719f 100644 --- a/backend/models.py +++ b/backend/models.py @@ -187,6 +187,10 @@ class Holiday(Base): source = Column(String(50), nullable=True) # "gov_hk_1823" / "manual" notes = Column(Text, nullable=True) + # Manual edit tracking - upload skips records with non-null last_edited_at + last_edited_at = Column(DateTime, nullable=True) + last_edited_by = Column(Integer, ForeignKey("users.id"), nullable=True) + created_at = Column(DateTime, default=now_hkt) updated_at = Column(DateTime, default=now_hkt, onupdate=now_hkt) @@ -210,6 +214,10 @@ class LeaveRecord(Base): source = Column(String(50), nullable=True) # "csv_upload" / "manual" notes = Column(Text, nullable=True) + # Manual edit tracking - upload skips records with non-null last_edited_at + last_edited_at = Column(DateTime, nullable=True) + last_edited_by = Column(Integer, ForeignKey("users.id"), nullable=True) + created_by = Column(Integer, ForeignKey("users.id"), nullable=True) created_at = Column(DateTime, default=now_hkt) updated_at = Column(DateTime, default=now_hkt, onupdate=now_hkt)