Delete VPS document stage after verified local storage sync

alembic/versions/20260720_document_vps_auto_cleanup.py

@@ -0,0 +1,42 @@
+"""Add VPS staging cleanup audit fields to document storage jobs.
+
+Revision ID: 20260720_document_vps_auto_cleanup
+Revises: 20260720_phase5c_workflow_escalations
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "20260720_document_vps_auto_cleanup"
+down_revision = "20260720_phase5c_workflow_escalations"
+branch_labels = None
+depends_on = None
+
+
+def _add_cleanup_columns(table_name: str) -> None:
+    op.add_column(table_name, sa.Column("vps_cleanup_status", sa.String(length=30), nullable=False, server_default="pending"))
+    op.add_column(table_name, sa.Column("vps_cleanup_attempts", sa.Integer(), nullable=False, server_default="0"))
+    op.add_column(table_name, sa.Column("vps_cleanup_error", sa.Text(), nullable=True))
+    op.add_column(table_name, sa.Column("vps_deleted_at_utc", sa.DateTime(timezone=True), nullable=True))
+    op.add_column(table_name, sa.Column("vps_deleted_bytes", sa.Integer(), nullable=True))
+    op.create_index(f"ix_{table_name}_vps_cleanup_status", table_name, ["vps_cleanup_status"], unique=False)
+    op.create_index(f"ix_{table_name}_vps_deleted_at_utc", table_name, ["vps_deleted_at_utc"], unique=False)
+
+
+def _drop_cleanup_columns(table_name: str) -> None:
+    op.drop_index(f"ix_{table_name}_vps_deleted_at_utc", table_name=table_name)
+    op.drop_index(f"ix_{table_name}_vps_cleanup_status", table_name=table_name)
+    op.drop_column(table_name, "vps_deleted_bytes")
+    op.drop_column(table_name, "vps_deleted_at_utc")
+    op.drop_column(table_name, "vps_cleanup_error")
+    op.drop_column(table_name, "vps_cleanup_attempts")
+    op.drop_column(table_name, "vps_cleanup_status")
+
+
+def upgrade() -> None:
+    _add_cleanup_columns("document_storage_jobs")
+    _add_cleanup_columns("permanent_document_storage_jobs")
+
+
+def downgrade() -> None:
+    _drop_cleanup_columns("permanent_document_storage_jobs")
+    _drop_cleanup_columns("document_storage_jobs")

alembic/versions/20260720_phase5c_workflow_escalations.py

@@ -0,0 +1,15 @@
+"""Add durable workflow escalation register.
+
+Revision ID: 20260720_phase5c_workflow_escalations
+Revises: 20260719_employee_workflow_phase3
+"""
+from alembic import op
+import sqlalchemy as sa
+revision="20260720_phase5c_workflow_escalations"
+down_revision="20260719_employee_workflow_phase3"
+branch_labels=None
+depends_on=None
+def upgrade():
+    op.create_table("workflow_escalations",sa.Column("id",sa.Integer(),primary_key=True),sa.Column("tenant_id",sa.Integer(),nullable=False),sa.Column("branch_id",sa.Integer(),nullable=True),sa.Column("subscription_id",sa.Integer(),nullable=False),sa.Column("task_id",sa.Integer(),nullable=True),sa.Column("raised_by_user_id",sa.Integer(),nullable=False),sa.Column("assigned_to_user_id",sa.Integer(),nullable=False),sa.Column("escalation_level",sa.String(30),nullable=False),sa.Column("category",sa.String(50),nullable=False,server_default="workflow_dependency"),sa.Column("priority",sa.String(20),nullable=False,server_default="high"),sa.Column("status",sa.String(20),nullable=False,server_default="open"),sa.Column("message",sa.Text(),nullable=False),sa.Column("follow_up_date",sa.Date(),nullable=True),sa.Column("acknowledged_at_utc",sa.DateTime(timezone=True),nullable=True),sa.Column("acknowledged_by_user_id",sa.Integer(),nullable=True),sa.Column("resolved_at_utc",sa.DateTime(timezone=True),nullable=True),sa.Column("resolved_by_user_id",sa.Integer(),nullable=True),sa.Column("resolution_note",sa.Text(),nullable=True),sa.Column("created_at_utc",sa.DateTime(timezone=True),nullable=False),sa.Column("updated_at_utc",sa.DateTime(timezone=True),nullable=False),sa.ForeignKeyConstraint(["tenant_id"],["tenants.id"],ondelete="CASCADE"),sa.ForeignKeyConstraint(["branch_id"],["branches.id"],ondelete="SET NULL"),sa.ForeignKeyConstraint(["subscription_id"],["client_service_subscriptions.id"],ondelete="CASCADE"),sa.ForeignKeyConstraint(["task_id"],["client_service_task_instances.id"],ondelete="SET NULL"),sa.ForeignKeyConstraint(["raised_by_user_id"],["users.id"],ondelete="RESTRICT"),sa.ForeignKeyConstraint(["assigned_to_user_id"],["users.id"],ondelete="RESTRICT"),sa.ForeignKeyConstraint(["acknowledged_by_user_id"],["users.id"],ondelete="SET NULL"),sa.ForeignKeyConstraint(["resolved_by_user_id"],["users.id"],ondelete="SET NULL"))
+    for c in ["tenant_id","branch_id","subscription_id","task_id","raised_by_user_id","assigned_to_user_id","escalation_level","category","priority","status","follow_up_date","created_at_utc"]: op.create_index(f"ix_workflow_escalations_{c}","workflow_escalations",[c])
+def downgrade(): op.drop_table("workflow_escalations")

app/modules/alerts/models.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from datetime import datetime, timezone
 
-from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Integer, String, Text
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.core.db.common import CommonBase
@@ -39,3 +39,37 @@ class UserAlert(CommonBase):
 
     user = relationship("User", foreign_keys=[user_id])
     created_by = relationship("User", foreign_keys=[created_by_user_id])
+
+
+class WorkflowEscalation(CommonBase):
+    """Durable engagement escalation register used by staff, managers and partners."""
+
+    __tablename__ = "workflow_escalations"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    tenant_id: Mapped[int] = mapped_column(ForeignKey("tenants.id", ondelete="CASCADE"), nullable=False, index=True)
+    branch_id: Mapped[int | None] = mapped_column(ForeignKey("branches.id", ondelete="SET NULL"), nullable=True, index=True)
+    subscription_id: Mapped[int] = mapped_column(ForeignKey("client_service_subscriptions.id", ondelete="CASCADE"), nullable=False, index=True)
+    task_id: Mapped[int | None] = mapped_column(ForeignKey("client_service_task_instances.id", ondelete="SET NULL"), nullable=True, index=True)
+    raised_by_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"), nullable=False, index=True)
+    assigned_to_user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="RESTRICT"), nullable=False, index=True)
+    escalation_level: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
+    category: Mapped[str] = mapped_column(String(50), nullable=False, default="workflow_dependency", index=True)
+    priority: Mapped[str] = mapped_column(String(20), nullable=False, default="high", index=True)
+    status: Mapped[str] = mapped_column(String(20), nullable=False, default="open", index=True)
+    message: Mapped[str] = mapped_column(Text, nullable=False)
+    follow_up_date: Mapped[date | None] = mapped_column(Date, nullable=True, index=True)
+    acknowledged_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    acknowledged_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    resolved_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    resolved_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    resolution_note: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True)
+    updated_at_utc: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), nullable=False)
+
+    subscription = relationship("ClientServiceSubscription")
+    task = relationship("ClientServiceTaskInstance")
+    raised_by = relationship("User", foreign_keys=[raised_by_user_id])
+    assigned_to = relationship("User", foreign_keys=[assigned_to_user_id])
+    acknowledged_by = relationship("User", foreign_keys=[acknowledged_by_user_id])
+    resolved_by = relationship("User", foreign_keys=[resolved_by_user_id])

app/modules/alerts/workflow_escalations.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from datetime import date, datetime, timezone
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session, selectinload
+
+from app.modules.alerts.models import WorkflowEscalation
+from app.modules.services.models import ClientServiceSubscription
+
+OPEN_ESCALATION_STATUSES = {"open", "acknowledged"}
+
+def create_workflow_escalation(db: Session, *, subscription: ClientServiceSubscription, raised_by_user_id: int, assigned_to_user_id: int, escalation_level: str, message: str, task_id: int | None = None, category: str = "workflow_dependency", priority: str = "high", follow_up_date: date | None = None) -> WorkflowEscalation:
+    row = WorkflowEscalation(tenant_id=subscription.tenant_id, branch_id=subscription.branch_id, subscription_id=subscription.id, task_id=task_id, raised_by_user_id=raised_by_user_id, assigned_to_user_id=assigned_to_user_id, escalation_level=escalation_level, category=category, priority=priority, status="open", message=message.strip()[:4000], follow_up_date=follow_up_date)
+    db.add(row)
+    db.flush()
+    return row
+
+def list_workflow_escalations(db: Session, *, tenant_id: int | None, branch_id: int | None, assigned_to_user_id: int | None = None, include_resolved: bool = False, limit: int = 200) -> list[dict[str, Any]]:
+    stmt = select(WorkflowEscalation).options(selectinload(WorkflowEscalation.subscription).selectinload(ClientServiceSubscription.client), selectinload(WorkflowEscalation.subscription).selectinload(ClientServiceSubscription.catalogue), selectinload(WorkflowEscalation.raised_by), selectinload(WorkflowEscalation.assigned_to))
+    if tenant_id: stmt = stmt.where(WorkflowEscalation.tenant_id == tenant_id)
+    if branch_id is not None: stmt = stmt.where(WorkflowEscalation.branch_id == branch_id)
+    if assigned_to_user_id: stmt = stmt.where(WorkflowEscalation.assigned_to_user_id == assigned_to_user_id)
+    if not include_resolved: stmt = stmt.where(WorkflowEscalation.status.in_(OPEN_ESCALATION_STATUSES))
+    rows = db.execute(stmt.order_by(WorkflowEscalation.created_at_utc.asc()).limit(limit)).scalars().all()
+    now = datetime.now(timezone.utc)
+    out=[]
+    for row in rows:
+        created=row.created_at_utc
+        if created and created.tzinfo is None: created=created.replace(tzinfo=timezone.utc)
+        sub=row.subscription
+        out.append({"id":row.id,"subscription_id":row.subscription_id,"task_id":row.task_id,"client_name":getattr(getattr(sub,"client",None),"client_name",None) or "Unlinked Client","service_name":getattr(getattr(sub,"catalogue",None),"service_name",None) or "Service","financial_year":getattr(sub,"financial_year",None) or "-","level":row.escalation_level,"category":row.category,"priority":row.priority,"status":row.status,"message":row.message,"follow_up_date":row.follow_up_date,"raised_by":getattr(row.raised_by,"full_name",None) or getattr(row.raised_by,"email",None) or "User","assigned_to":getattr(row.assigned_to,"full_name",None) or getattr(row.assigned_to,"email",None) or "User","created_at":row.created_at_utc,"age_days":max(0,(now-created).days) if created else 0,"href":f"/manager/reviews/engagements/{row.subscription_id}" if row.escalation_level=="manager" else f"/partner/reviews/engagements/{row.subscription_id}"})
+    return out
+
+def update_workflow_escalation(db: Session, *, escalation_id: int, actor_user_id: int, action: str, resolution_note: str = "") -> WorkflowEscalation:
+    row=db.get(WorkflowEscalation, escalation_id)
+    if not row or int(row.assigned_to_user_id)!=int(actor_user_id): raise ValueError("Escalation not found or not assigned to you.")
+    now=datetime.now(timezone.utc)
+    if action=="acknowledge": row.status="acknowledged"; row.acknowledged_at_utc=now; row.acknowledged_by_user_id=actor_user_id
+    elif action=="resolve": row.status="resolved"; row.resolved_at_utc=now; row.resolved_by_user_id=actor_user_id; row.resolution_note=(resolution_note or "").strip()[:4000] or None
+    else: raise ValueError("Invalid escalation action.")
+    db.flush(); return row

app/modules/documents/models.py

@@ -213,6 +213,15 @@ class DocumentStorageJob(CommonBase):
     completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     acknowledged_hash_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
     local_final_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
+
+    # VPS staging cleanup audit. The file is deleted only after the branch agent
+    # acknowledges the exact expected SHA-256 hash.
+    vps_cleanup_status: Mapped[str] = mapped_column(String(30), nullable=False, default="pending", index=True)
+    vps_cleanup_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    vps_cleanup_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    vps_deleted_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+    vps_deleted_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+
     created_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
 
     node = relationship("BranchStorageNode")
@@ -393,6 +402,15 @@ class PermanentDocumentStorageJob(CommonBase):
     completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     acknowledged_hash_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
     local_final_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
+
+    # VPS staging cleanup audit. The file is deleted only after the branch agent
+    # acknowledges the exact expected SHA-256 hash.
+    vps_cleanup_status: Mapped[str] = mapped_column(String(30), nullable=False, default="pending", index=True)
+    vps_cleanup_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    vps_cleanup_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    vps_deleted_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
+    vps_deleted_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+
     created_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
 
     node = relationship("BranchStorageNode")

app/modules/documents/services.py

@@ -53,6 +53,11 @@ UDIN_REQUIRED_DOCUMENT_TYPES = {"AUDIT_REPORT", "SIGNED_OUTPUT", "ACKNOWLEDGEMEN
 
 MAX_UPLOAD_BYTES = int(os.getenv("DOCUMENT_MAX_UPLOAD_MB", "50")) * 1024 * 1024
 
+# A server copy is retained only as a temporary transfer buffer. It is deleted
+# after the local storage agent confirms the exact expected SHA-256 hash.
+# Emergency rollback: DOCUMENT_DELETE_VPS_AFTER_LOCAL_SYNC=false
+DELETE_VPS_AFTER_LOCAL_SYNC = (os.getenv("DOCUMENT_DELETE_VPS_AFTER_LOCAL_SYNC", "true").strip().lower() not in {"0", "false", "no", "off"})
+
 
 def _resolve_document_storage_root(env_name: str, default_leaf: str) -> Path:
     """Return a short absolute storage root.
@@ -815,6 +820,139 @@ def acknowledge_storage_job(db: Session, *, node: BranchStorageNode, job: Docume
     return False
 
 
+def _prune_empty_stage_directories(file_path: Path, root: Path) -> None:
+    """Remove empty parent folders without crossing the configured root."""
+    root = root.resolve()
+    parent = file_path.parent
+    while parent != root:
+        try:
+            parent.rmdir()
+        except OSError:
+            break
+        parent = parent.parent
+
+
+def _verified_stage_path(version) -> Path | None:
+    """Resolve the staged path and reject traversal outside storage root."""
+    if not version or not getattr(version, "local_relative_path", None):
+        return None
+    root = DEFAULT_STORAGE_ROOT.resolve()
+    candidate = (DEFAULT_STORAGE_ROOT / Path(version.local_relative_path)).resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError:
+        return None
+    return candidate
+
+
+def _cleanup_verified_vps_stage(*, job, version) -> dict[str, object]:
+    """Delete one VPS stage only after verified branch-local storage."""
+    result: dict[str, object] = {"enabled": DELETE_VPS_AFTER_LOCAL_SYNC, "deleted": False}
+    if not DELETE_VPS_AFTER_LOCAL_SYNC:
+        result["reason"] = "cleanup_disabled"
+        return result
+    if not job or job.status != "completed":
+        result["reason"] = "job_not_completed"
+        return result
+    if getattr(job, "vps_cleanup_status", "pending") == "deleted":
+        result.update({"deleted": True, "already_completed": True, "reason": "already_deleted"})
+        return result
+
+    expected = (job.expected_hash_sha256 or "").strip().lower()
+    acknowledged = (job.acknowledged_hash_sha256 or "").strip().lower()
+    if not expected or acknowledged != expected:
+        result["reason"] = "local_hash_not_verified"
+        return result
+    if not (job.local_final_path or "").strip():
+        result["reason"] = "local_final_path_missing"
+        return result
+
+    job.vps_cleanup_attempts = int(job.vps_cleanup_attempts or 0) + 1
+    job.vps_cleanup_status = "deleting"
+    job.vps_cleanup_error = None
+    stage_path = _verified_stage_path(version)
+    if stage_path is None:
+        job.vps_cleanup_status = "failed"
+        job.vps_cleanup_error = "Unsafe or missing staged path."
+        result["reason"] = "unsafe_stage_path"
+        return result
+
+    try:
+        deleted_bytes = int(stage_path.stat().st_size) if stage_path.exists() and stage_path.is_file() else 0
+        if stage_path.exists():
+            if not stage_path.is_file():
+                raise OSError("Staged path is not a regular file.")
+            stage_path.unlink()
+            _prune_empty_stage_directories(stage_path, DEFAULT_STORAGE_ROOT)
+        job.vps_cleanup_status = "deleted"
+        job.vps_cleanup_error = None
+        job.vps_deleted_at_utc = _now_utc()
+        job.vps_deleted_bytes = deleted_bytes
+        result.update({
+            "deleted": True,
+            "already_missing": deleted_bytes == 0,
+            "deleted_bytes": deleted_bytes,
+            "reason": "deleted_after_verified_local_sync",
+        })
+        return result
+    except OSError as exc:
+        job.vps_cleanup_status = "retry" if int(job.vps_cleanup_attempts or 0) < 20 else "failed"
+        job.vps_cleanup_error = f"{type(exc).__name__}: {str(exc)[:900]}"
+        result.update({"reason": "delete_failed", "error": job.vps_cleanup_error})
+        return result
+
+
+def cleanup_completed_storage_job_vps_stage(db: Session, job: DocumentStorageJob) -> dict[str, object]:
+    version = db.get(EngagementDocumentVersion, job.version_id) if job else None
+    return _cleanup_verified_vps_stage(job=job, version=version)
+
+
+def cleanup_completed_permanent_storage_job_vps_stage(db: Session, job: PermanentDocumentStorageJob) -> dict[str, object]:
+    version = db.get(PermanentClientDocumentVersion, job.version_id) if job else None
+    return _cleanup_verified_vps_stage(job=job, version=version)
+
+
+def retry_verified_vps_cleanup_for_node(db: Session, node: BranchStorageNode, limit: int = 50) -> dict[str, int]:
+    """Retry interrupted/failed deletion during heartbeat or tunnel sync."""
+    summary = {"checked": 0, "deleted": 0, "failed": 0}
+    if not DELETE_VPS_AFTER_LOCAL_SYNC or not node:
+        return summary
+    retry_states = ["pending", "retry", "failed", "deleting"]
+    engagement_jobs = db.execute(
+        select(DocumentStorageJob)
+        .where(
+            DocumentStorageJob.storage_node_id == node.id,
+            DocumentStorageJob.status == "completed",
+            DocumentStorageJob.vps_cleanup_status.in_(retry_states),
+        )
+        .order_by(DocumentStorageJob.completed_at_utc.asc(), DocumentStorageJob.id.asc())
+        .limit(limit)
+    ).scalars().all()
+    remaining = max(0, limit - len(engagement_jobs))
+    permanent_jobs = []
+    if remaining:
+        permanent_jobs = db.execute(
+            select(PermanentDocumentStorageJob)
+            .where(
+                PermanentDocumentStorageJob.storage_node_id == node.id,
+                PermanentDocumentStorageJob.status == "completed",
+                PermanentDocumentStorageJob.vps_cleanup_status.in_(retry_states),
+            )
+            .order_by(PermanentDocumentStorageJob.completed_at_utc.asc(), PermanentDocumentStorageJob.id.asc())
+            .limit(remaining)
+        ).scalars().all()
+
+    for job in engagement_jobs:
+        summary["checked"] += 1
+        result = cleanup_completed_storage_job_vps_stage(db, job)
+        summary["deleted" if result.get("deleted") else "failed"] += 1
+    for job in permanent_jobs:
+        summary["checked"] += 1
+        result = cleanup_completed_permanent_storage_job_vps_stage(db, job)
+        summary["deleted" if result.get("deleted") else "failed"] += 1
+    return summary
+
+
 def list_storage_jobs(db: Session, tenant_id: int | None = None, branch_id: int | None = None, status: str | None = None, limit: int = 200):
     stmt = select(DocumentStorageJob).order_by(DocumentStorageJob.created_at_utc.desc())
     if tenant_id:

app/modules/documents/ui.py

@@ -27,6 +27,9 @@ from app.modules.documents.services import (
     get_latest_version,
     get_version,
     acknowledge_storage_job,
+    cleanup_completed_storage_job_vps_stage,
+    cleanup_completed_permanent_storage_job_vps_stage,
+    retry_verified_vps_cleanup_for_node,
     authenticate_storage_node,
     create_branch_storage_node,
     generate_storage_secret,
@@ -1568,8 +1571,9 @@ async def storage_agent_heartbeat(request: Request, x_node_code: str | None = He
         node, error = _agent_auth(db, request, x_node_code, x_node_secret)
         if error:
             return error
+        cleanup = retry_verified_vps_cleanup_for_node(db, node, limit=50)
         db.commit()
-        return {"ok": True, "node_code": node.node_code, "status": node.status}
+        return {"ok": True, "node_code": node.node_code, "status": node.status, "vps_cleanup": cleanup}
     finally:
         db.close()
 
@@ -1644,6 +1648,8 @@ def _permanent_download_requests_payload(items):
 
 
 def _storage_agent_sync_payload(db, node):
+    # Retry cleanup in tunnel mode as well as normal heartbeat mode.
+    retry_verified_vps_cleanup_for_node(db, node, limit=25)
     jobs = _normal_storage_jobs_payload(list_pending_storage_jobs(db, node))
     jobs += _permanent_storage_jobs_payload(list_pending_permanent_storage_jobs(db, node))
     requests_ = _normal_download_requests_payload(list_pending_download_requests(db, node))
@@ -1719,8 +1725,17 @@ async def storage_agent_ack_job(request: Request, job_id: str, x_node_code: str
             if not job:
                 return JSONResponse({"ok": False, "error": "job_not_found"}, status_code=404)
             ok = acknowledge_storage_job(db, node=node, job=job, acknowledged_hash=acknowledged_hash, local_final_path=local_final_path, success=bool(payload.get("success", True)), error=payload.get("error"))
+        # First commit the verified local acknowledgement. Only after that
+        # durable commit is the VPS staging file eligible for deletion.
         db.commit()
-        return {"ok": ok, "job_status": job.status}
+        cleanup = {"enabled": True, "deleted": False, "reason": "acknowledgement_failed"}
+        if ok:
+            if is_permanent:
+                cleanup = cleanup_completed_permanent_storage_job_vps_stage(db, job)
+            else:
+                cleanup = cleanup_completed_storage_job_vps_stage(db, job)
+            db.commit()
+        return {"ok": ok, "job_status": job.status, "vps_cleanup": cleanup}
     finally:
         db.close()
 

app/modules/employees/service.py

@@ -14,6 +14,7 @@ from sqlalchemy.orm import Session, selectinload
 from app.core.security.passwords import hash_password
 from app.modules.core.iam.models import User
 from app.modules.alerts.service import create_alert
+from app.modules.alerts.workflow_escalations import create_workflow_escalation
 from app.modules.core.iam.scope import build_scope, list_visible_branches, list_visible_tenants, validate_branch_matches_tenant
 from app.modules.core.rbac.deps import get_user_roles
 from app.modules.core.rbac.models import Role, UserRole
@@ -3255,6 +3256,11 @@ def escalate_employee_engagement_workflow(
     clean_message = (message or "").strip()[:1000]
     if not clean_message:
         raise ValueError("Escalation details are required.")
+    create_workflow_escalation(
+        db, subscription=subscription, raised_by_user_id=actor_user_id,
+        assigned_to_user_id=int(target_id), escalation_level=code,
+        message=clean_message, category="workflow_dependency", priority="high",
+    )
     create_alert(
         db,
         user_id=int(target_id),

app/modules/manager_dashboard/service.py

@@ -7,6 +7,7 @@ from sqlalchemy import func, or_, select
 from sqlalchemy.orm import Session, selectinload
 
 from app.modules.clients.models import Client
+from app.modules.alerts.workflow_escalations import list_workflow_escalations
 from app.modules.core.iam.models import User
 from app.modules.core.rbac.models import Permission, Role, RolePermission, UserRole
 from app.modules.employees.service import (
@@ -385,6 +386,7 @@ def build_manager_dashboard_payload(db: Session, request, current_user) -> dict[
     staff_rows = _staff_rows(db, scope, task_rows)
     client_rows = _client_rows(db, scope, task_rows)
     advanced_engagements = _advanced_engagement_rows(tasks)
+    unified_escalations = list_workflow_escalations(db, tenant_id=getattr(scope, "tenant_id", None), branch_id=getattr(scope, "branch_id", None), assigned_to_user_id=current_user.id)
     capacity_rows = _capacity_rows(tasks)
     sla_breached = [row for row in advanced_engagements if row["sla"]["status"] == "breached"]
     sla_warning = [row for row in advanced_engagements if row["sla"]["status"] in {"critical", "warning"}]
@@ -430,6 +432,8 @@ def build_manager_dashboard_payload(db: Session, request, current_user) -> dict[
         "staff_count": len(staff_rows),
         "client_count": len(client_rows),
         "escalation_count": len(escalation_rows),
+        "unified_escalation_count": len(unified_escalations),
+        "escalations_over_3_days": len([r for r in unified_escalations if r["age_days"] >= 3]),
         "engagement_count": len(advanced_engagements),
         "sla_breached_count": len(sla_breached),
         "sla_warning_count": len(sla_warning),
@@ -451,6 +455,7 @@ def build_manager_dashboard_payload(db: Session, request, current_user) -> dict[
         "client_pending": client_pending_rows[:60],
         "documents_pending": client_pending_rows[:40],
         "escalations": escalation_rows[:80],
+        "unified_escalations": unified_escalations,
         "age_buckets": age_buckets,
         "status_summary": sorted([{"label": k, "count": v} for k, v in status_summary.items()], key=lambda x: x["count"], reverse=True)[:10],
         "service_summary": sorted([{"label": k, "count": v} for k, v in service_summary.items()], key=lambda x: x["count"], reverse=True)[:10],

app/modules/manager_dashboard/templates/manager_dashboard/partials/escalations.html

@@ -1,5 +1,5 @@
 <div class="space-y-6">
-  <section class="grid gap-4 md:grid-cols-2 xl:grid-cols-4"><div class="af-metric-card border-red-200 bg-red-50"><div class="text-xs font-semibold uppercase text-red-700">SLA Breached</div><div class="mt-2 text-3xl font-semibold text-red-700">{{ overview.sla_breached_count }}</div></div><div class="af-metric-card border-amber-200 bg-amber-50"><div class="text-xs font-semibold uppercase text-amber-700">SLA Warning</div><div class="mt-2 text-3xl font-semibold text-amber-700">{{ overview.sla_warning_count }}</div></div><div class="af-metric-card"><div class="text-xs font-semibold uppercase text-slate-500">Task Escalations</div><div class="mt-2 text-3xl font-semibold">{{ overview.escalation_count }}</div></div><div class="af-metric-card border-violet-200 bg-violet-50"><div class="text-xs font-semibold uppercase text-violet-700">Review Dependencies</div><div class="mt-2 text-3xl font-semibold text-violet-700">{{ overview.engagement_review_count }}</div></div></section>
+<section class="grid gap-4 md:grid-cols-3"><div class="af-metric-card border-violet-200 bg-violet-50"><div class="text-xs font-semibold uppercase text-violet-700">Open Escalations</div><div class="mt-2 text-3xl font-semibold text-violet-700">{{ overview.unified_escalation_count }}</div></div><div class="af-metric-card border-red-200 bg-red-50"><div class="text-xs font-semibold uppercase text-red-700">Age 3+ Days</div><div class="mt-2 text-3xl font-semibold text-red-700">{{ overview.escalations_over_3_days }}</div></div><div class="af-metric-card"><div class="text-xs font-semibold uppercase text-slate-500">SLA Breached</div><div class="mt-2 text-3xl font-semibold">{{ overview.sla_breached_count }}</div></div></section>
-  <section class="grid gap-6 xl:grid-cols-2"><div class="af-card overflow-x-auto"><h3 class="text-lg font-semibold">SLA Breaches</h3><p class="mb-4 text-sm text-slate-500">Engagement-level statutory/internal due-date breaches.</p><table class="min-w-full text-sm"><tbody>{% for row in sla_breached %}<tr class="border-b"><td class="px-3 py-3"><a href="{{ row.href }}" class="font-semibold text-brand-700">{{ row.client_name }} — {{ row.service_name }}</a><div class="text-xs text-slate-500">{{ row.financial_year }} · {{ row.progress_percent }}% complete</div></td><td class="px-3 py-3 text-right"><div class="font-semibold text-red-700">{{ row.sla.label }}</div><div class="text-xs text-slate-500">Age {{ row.sla.age_days }} day(s)</div></td></tr>{% else %}<tr><td class="px-3 py-8 text-center text-slate-500">No SLA breaches.</td></tr>{% endfor %}</tbody></table></div><div class="af-card overflow-x-auto"><h3 class="text-lg font-semibold">SLA Warning Window</h3><p class="mb-4 text-sm text-slate-500">Due within seven days.</p><table class="min-w-full text-sm"><tbody>{% for row in sla_warning %}<tr class="border-b"><td class="px-3 py-3"><a href="{{ row.href }}" class="font-semibold text-brand-700">{{ row.client_name }} — {{ row.service_name }}</a><div class="text-xs text-slate-500">{{ row.financial_year }} · {{ row.progress_percent }}% complete</div></td><td class="px-3 py-3 text-right font-semibold text-amber-700">{{ row.sla.label }}</td></tr>{% else %}<tr><td class="px-3 py-8 text-center text-slate-500">No engagements in warning window.</td></tr>{% endfor %}</tbody></table></div></section>
+<section class="af-card overflow-x-auto"><h3 class="text-lg font-semibold">Unified Escalation Register</h3><p class="mb-4 text-sm text-slate-500">Durable staff-to-manager escalations with acknowledgement, resolution and ageing.</p><table class="min-w-full text-sm"><thead><tr class="border-b text-left text-xs uppercase text-slate-500"><th class="px-3 py-2">Engagement</th><th class="px-3 py-2">Escalation</th><th class="px-3 py-2">Age</th><th class="px-3 py-2">Action</th></tr></thead><tbody>{% for row in unified_escalations %}<tr class="border-b align-top"><td class="px-3 py-3"><a href="{{ row.href }}" class="font-semibold text-brand-700">{{ row.client_name }} — {{ row.service_name }}</a><div class="text-xs text-slate-500">{{ row.financial_year }} · Raised by {{ row.raised_by }}</div></td><td class="px-3 py-3"><div class="font-semibold">{{ row.category|replace('_',' ')|title }}</div><div class="mt-1 max-w-xl text-slate-600">{{ row.message }}</div><div class="mt-1 text-xs text-slate-500">Status: {{ row.status|title }}{% if row.follow_up_date %} · Follow-up {{ row.follow_up_date }}{% endif %}</div></td><td class="px-3 py-3"><span class="rounded-full px-2 py-1 text-xs font-semibold {% if row.age_days >= 3 %}bg-red-100 text-red-700{% else %}bg-slate-100 text-slate-700{% endif %}">{{ row.age_days }} day(s)</span></td><td class="px-3 py-3"><form method="post" action="/manager/escalations/{{ row.id }}/action" class="space-y-2"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><textarea name="resolution_note" rows="2" class="w-56 rounded-lg border px-2 py-1 text-xs" placeholder="Resolution note"></textarea><div class="flex gap-2"><button name="action" value="acknowledge" class="af-btn af-btn-secondary text-xs">Acknowledge</button><button name="action" value="resolve" class="af-btn af-btn-primary text-xs">Resolve</button></div></form></td></tr>{% else %}<tr><td colspan="4" class="px-3 py-8 text-center text-slate-500">No open escalations assigned to you.</td></tr>{% endfor %}</tbody></table></section>
-  <div class="af-card"><div class="mb-4 flex items-center justify-between"><div><h3 class="text-lg font-semibold">Existing Task Escalation Register</h3><p class="text-sm text-slate-500">Unassigned, overdue and client-pending tasks remain visible.</p></div><a href="/alerts" class="af-btn af-btn-primary">Open Alerts</a></div>{% set rows = escalations %}{% include 'modules/manager_dashboard/templates/manager_dashboard/partials/_task_table.html' %}</div>
+<section class="grid gap-6 xl:grid-cols-2"><div class="af-card"><h3 class="text-lg font-semibold">Review Ageing — SLA Breaches</h3><div class="mt-3 space-y-3">{% for row in sla_breached %}<a href="{{ row.href }}" class="block rounded-xl border p-3"><div class="font-semibold">{{ row.client_name }} — {{ row.service_name }}</div><div class="text-xs text-red-700">{{ row.sla.label }} · Age {{ row.sla.age_days }} day(s)</div></a>{% else %}<div class="text-sm text-slate-500">No breached engagements.</div>{% endfor %}</div></div><div class="af-card"><h3 class="text-lg font-semibold">Review Ageing — Warning</h3><div class="mt-3 space-y-3">{% for row in sla_warning %}<a href="{{ row.href }}" class="block rounded-xl border p-3"><div class="font-semibold">{{ row.client_name }} — {{ row.service_name }}</div><div class="text-xs text-amber-700">{{ row.sla.label }}</div></a>{% else %}<div class="text-sm text-slate-500">No warning-window engagements.</div>{% endfor %}</div></div></section>
-</div>
+</div>

app/modules/manager_dashboard/ui.py

@@ -16,6 +16,7 @@ from app.modules.manager_dashboard.service import (
     get_next_manager_review_task_id,
 )
 from app.modules.services.execution import apply_task_review
+from app.modules.alerts.workflow_escalations import update_workflow_escalation
 
 router = APIRouter(prefix="/manager", tags=["manager-dashboard-v2-ui"])
 
@@ -180,3 +181,17 @@ def manager_review_task_submit(
         return RedirectResponse(url=f"/manager/reviews/engagements/{subscription_id}?task_id={task_id}&reviewed=1", status_code=303)
     finally:
         db.close()
+
+
+@router.post("/escalations/{escalation_id}/action")
+def manager_escalation_action(request: Request, escalation_id: int, action: str = Form(...), resolution_note: str = Form(""), csrf_token: str = Form(...)):
+    validate_csrf(request, csrf_token)
+    db=CommonSessionLocal()
+    try:
+        current_user=get_current_user(request, db=db)
+        if not current_user: return RedirectResponse(url="/login",status_code=303)
+        if not can_access_manager_dashboard(db,current_user): return ui_access_denied()
+        try: update_workflow_escalation(db, escalation_id=escalation_id, actor_user_id=current_user.id, action=action, resolution_note=resolution_note); db.commit()
+        except ValueError: db.rollback()
+        return RedirectResponse(url="/manager/dashboard?tab=escalations",status_code=303)
+    finally: db.close()

app/modules/partner_dashboard/service.py

@@ -8,6 +8,7 @@ from sqlalchemy import func, or_, select
 from sqlalchemy.orm import Session, selectinload
 
 from app.modules.clients.models import Client
+from app.modules.alerts.workflow_escalations import list_workflow_escalations
 from app.modules.core.iam.models import User
 from app.modules.core.rbac.models import Role, UserRole
 from app.modules.core.tenancy.models import Branch, Tenant
@@ -297,6 +298,7 @@ def build_partner_dashboard_payload(db: Session, request, current_user) -> dict[
     clients = _load_clients(db, tenant_id, branch_id, current_user, roles)
     staff = _staff_rows(db, tenant_id, branch_id, tasks)
     billing = _billing_rows(db, tenant_id, branch_id, fy)
+    unified_escalations = list_workflow_escalations(db, tenant_id=tenant_id, branch_id=branch_id, assigned_to_user_id=current_user.id)
 
     subscriptions_count = _count(db, _subscription_scope(select(func.count(ClientServiceSubscription.id)), tenant_id, branch_id, current_user, roles, fy)) if tenant_id else 0
 
@@ -318,6 +320,8 @@ def build_partner_dashboard_payload(db: Session, request, current_user) -> dict[
         "invoice_count": billing["invoice_count"],
         "outstanding": billing["outstanding"],
         "today": today,
+        "unified_escalation_count": len(unified_escalations),
+        "escalations_over_3_days": len([r for r in unified_escalations if r["age_days"] >= 3]),
     }
 
     return {
@@ -330,6 +334,7 @@ def build_partner_dashboard_payload(db: Session, request, current_user) -> dict[
         "client_pending": client_pending_rows[:25],
         "review_queue": review_rows[:50],
         "partner_engagement_review_queue": _partner_review_queue_rows(db, request, current_user),
+        "unified_escalations": unified_escalations,
         "clients": clients,
         "staff_rows": staff,
         "billing": billing,

app/modules/partner_dashboard/templates/partner_dashboard/partials/escalations.html

@@ -0,0 +1 @@
+<div class="space-y-6"><section class="grid gap-4 md:grid-cols-3"><div class="af-metric-card border-violet-200 bg-violet-50"><div class="text-xs font-semibold uppercase text-violet-700">Open Escalations</div><div class="mt-2 text-3xl font-semibold text-violet-700">{{ overview.unified_escalation_count }}</div></div><div class="af-metric-card border-red-200 bg-red-50"><div class="text-xs font-semibold uppercase text-red-700">Age 3+ Days</div><div class="mt-2 text-3xl font-semibold text-red-700">{{ overview.escalations_over_3_days }}</div></div><div class="af-metric-card"><div class="text-xs font-semibold uppercase text-slate-500">Partner Reviews</div><div class="mt-2 text-3xl font-semibold">{{ overview.review_pending_count }}</div></div></section><section class="af-card overflow-x-auto"><h3 class="text-lg font-semibold">Unified Partner Escalation & Review Ageing</h3><p class="mb-4 text-sm text-slate-500">Escalations assigned to the Engagement Partner or Review Partner.</p><table class="min-w-full text-sm"><tbody>{% for row in unified_escalations %}<tr class="border-b align-top"><td class="px-3 py-3"><a href="{{ row.href }}" class="font-semibold text-brand-700">{{ row.client_name }} — {{ row.service_name }}</a><div class="text-xs text-slate-500">{{ row.financial_year }} · {{ row.level|replace('_',' ')|title }} · {{ row.age_days }} day(s)</div><div class="mt-2 text-slate-600">{{ row.message }}</div></td><td class="px-3 py-3 text-right"><form method="post" action="/partner/escalations/{{ row.id }}/action" class="inline-block space-y-2 text-left"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><textarea name="resolution_note" rows="2" class="w-56 rounded-lg border px-2 py-1 text-xs" placeholder="Resolution note"></textarea><div class="flex gap-2"><button name="action" value="acknowledge" class="af-btn af-btn-secondary text-xs">Acknowledge</button><button name="action" value="resolve" class="af-btn af-btn-primary text-xs">Resolve</button></div></form></td></tr>{% else %}<tr><td class="px-3 py-8 text-center text-slate-500">No open escalations assigned to you.</td></tr>{% endfor %}</tbody></table></section><section class="af-card"><h3 class="text-lg font-semibold">Partner Review Ageing</h3><div class="mt-3 space-y-3">{% for row in partner_engagement_review_queue %}<a href="{{ row.href }}" class="block rounded-xl border p-3"><div class="font-semibold">{{ row.client_name }} — {{ row.service_name }}</div><div class="text-xs text-slate-500">{{ row.financial_year }} · Partner {{ row.pending_partner }} · Review Partner {{ row.pending_review_partner }} · {{ row.sla.label }}</div></a>{% else %}<div class="text-sm text-slate-500">No pending partner reviews.</div>{% endfor %}</div></section></div>

app/modules/partner_dashboard/templates/partner_dashboard/partials/review.html

@@ -1,4 +1,5 @@
 <div class="space-y-6">
+  <div class="flex justify-end"><a href="/partner/dashboard?tab=escalations" class="af-btn af-btn-secondary">Escalations & Review Ageing</a></div>
   <div class="rounded-3xl border border-slate-200 bg-white p-5 shadow-soft">
     <div class="flex flex-col gap-3 lg:flex-row lg:items-center lg:justify-between">
       <div>

app/modules/partner_dashboard/ui.py

@@ -16,6 +16,7 @@ from app.modules.partner_dashboard.service import (
     get_partner_review_workspace,
 )
 from app.modules.services.execution import apply_task_review
+from app.modules.alerts.workflow_escalations import update_workflow_escalation
 
 router = APIRouter(prefix="/partner", tags=["partner-dashboard-v2-ui"])
 
@@ -25,6 +26,7 @@ VALID_TABS = {
     "clients": "modules/partner_dashboard/templates/partner_dashboard/partials/clients.html",
     "staff": "modules/partner_dashboard/templates/partner_dashboard/partials/staff.html",
     "review": "modules/partner_dashboard/templates/partner_dashboard/partials/review.html",
+    "escalations": "modules/partner_dashboard/templates/partner_dashboard/partials/escalations.html",
     "billing": "modules/partner_dashboard/templates/partner_dashboard/partials/billing.html",
     "reports": "modules/partner_dashboard/templates/partner_dashboard/partials/reports.html",
     "wizards": "modules/partner_dashboard/templates/partner_dashboard/partials/wizards.html",
@@ -168,3 +170,17 @@ def partner_review_task_submit(
         return RedirectResponse(url=f"/partner/reviews/engagements/{subscription_id}?task_id={task_id}&reviewed=1", status_code=303)
     finally:
         db.close()
+
+
+@router.post("/escalations/{escalation_id}/action")
+def partner_escalation_action(request: Request, escalation_id: int, action: str = Form(...), resolution_note: str = Form(""), csrf_token: str = Form(...)):
+    validate_csrf(request, csrf_token)
+    db=CommonSessionLocal()
+    try:
+        current_user=get_current_user(request, db=db)
+        if not current_user: return RedirectResponse(url="/login",status_code=303)
+        if not can_access_partner_dashboard(db,current_user): return ui_access_denied()
+        try: update_workflow_escalation(db, escalation_id=escalation_id, actor_user_id=current_user.id, action=action, resolution_note=resolution_note); db.commit()
+        except ValueError: db.rollback()
+        return RedirectResponse(url="/partner/dashboard?tab=escalations",status_code=303)
+    finally: db.close()