Delete VPS document stage after verified local storage sync

app/modules/documents/ui.py

@@ -27,6 +27,9 @@ from app.modules.documents.services import (
     get_latest_version,
     get_version,
     acknowledge_storage_job,
+    cleanup_completed_storage_job_vps_stage,
+    cleanup_completed_permanent_storage_job_vps_stage,
+    retry_verified_vps_cleanup_for_node,
     authenticate_storage_node,
     create_branch_storage_node,
     generate_storage_secret,
@@ -1568,8 +1571,9 @@ async def storage_agent_heartbeat(request: Request, x_node_code: str | None = He
         node, error = _agent_auth(db, request, x_node_code, x_node_secret)
         if error:
             return error
+        cleanup = retry_verified_vps_cleanup_for_node(db, node, limit=50)
         db.commit()
-        return {"ok": True, "node_code": node.node_code, "status": node.status}
+        return {"ok": True, "node_code": node.node_code, "status": node.status, "vps_cleanup": cleanup}
     finally:
         db.close()
 
@@ -1644,6 +1648,8 @@ def _permanent_download_requests_payload(items):
 
 
 def _storage_agent_sync_payload(db, node):
+    # Retry cleanup in tunnel mode as well as normal heartbeat mode.
+    retry_verified_vps_cleanup_for_node(db, node, limit=25)
     jobs = _normal_storage_jobs_payload(list_pending_storage_jobs(db, node))
     jobs += _permanent_storage_jobs_payload(list_pending_permanent_storage_jobs(db, node))
     requests_ = _normal_download_requests_payload(list_pending_download_requests(db, node))
@@ -1719,8 +1725,17 @@ async def storage_agent_ack_job(request: Request, job_id: str, x_node_code: str
             if not job:
                 return JSONResponse({"ok": False, "error": "job_not_found"}, status_code=404)
             ok = acknowledge_storage_job(db, node=node, job=job, acknowledged_hash=acknowledged_hash, local_final_path=local_final_path, success=bool(payload.get("success", True)), error=payload.get("error"))
+        # First commit the verified local acknowledgement. Only after that
+        # durable commit is the VPS staging file eligible for deletion.
         db.commit()
-        return {"ok": ok, "job_status": job.status}
+        cleanup = {"enabled": True, "deleted": False, "reason": "acknowledgement_failed"}
+        if ok:
+            if is_permanent:
+                cleanup = cleanup_completed_permanent_storage_job_vps_stage(db, job)
+            else:
+                cleanup = cleanup_completed_storage_job_vps_stage(db, job)
+            db.commit()
+        return {"ok": ok, "job_status": job.status, "vps_cleanup": cleanup}
     finally:
         db.close()