Rakseeibra/arrr-erp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Delete VPS document stage after verified local storage sync

Browse Source
This commit is contained in:
A R R R Associates
2026-07-20 15:27:58 +05:30
parent bca72ab2d9
commit 55c5a7acdf
15 changed files with 361 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/modules/documents/models.py
+18
View File
@@ -213,6 +213,15 @@ class DocumentStorageJob(CommonBase):
completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
acknowledged_hash_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
local_final_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
# VPS staging cleanup audit. The file is deleted only after the branch agent
# acknowledges the exact expected SHA-256 hash.
vps_cleanup_status: Mapped[str] = mapped_column(String(30), nullable=False, default="pending", index=True)
vps_cleanup_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
vps_cleanup_error: Mapped[str | None] = mapped_column(Text, nullable=True)
vps_deleted_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
vps_deleted_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
created_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
node = relationship("BranchStorageNode")
@@ -393,6 +402,15 @@ class PermanentDocumentStorageJob(CommonBase):
completed_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
acknowledged_hash_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
local_final_path: Mapped[str | None] = mapped_column(String(1000), nullable=True)
# VPS staging cleanup audit. The file is deleted only after the branch agent
# acknowledges the exact expected SHA-256 hash.
vps_cleanup_status: Mapped[str] = mapped_column(String(30), nullable=False, default="pending", index=True)
vps_cleanup_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
vps_cleanup_error: Mapped[str | None] = mapped_column(Text, nullable=True)
vps_deleted_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
vps_deleted_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
created_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
node = relationship("BranchStorageNode")
app/modules/documents/services.py
+138
View File
@@ -53,6 +53,11 @@ UDIN_REQUIRED_DOCUMENT_TYPES = {"AUDIT_REPORT", "SIGNED_OUTPUT", "ACKNOWLEDGEMEN
MAX_UPLOAD_BYTES = int(os.getenv("DOCUMENT_MAX_UPLOAD_MB", "50")) * 1024 * 1024
# A server copy is retained only as a temporary transfer buffer. It is deleted
# after the local storage agent confirms the exact expected SHA-256 hash.
# Emergency rollback: DOCUMENT_DELETE_VPS_AFTER_LOCAL_SYNC=false
DELETE_VPS_AFTER_LOCAL_SYNC = (os.getenv("DOCUMENT_DELETE_VPS_AFTER_LOCAL_SYNC", "true").strip().lower() not in {"0", "false", "no", "off"})
def _resolve_document_storage_root(env_name: str, default_leaf: str) -> Path:
"""Return a short absolute storage root.
@@ -815,6 +820,139 @@ def acknowledge_storage_job(db: Session, *, node: BranchStorageNode, job: Docume
return False
def _prune_empty_stage_directories(file_path: Path, root: Path) -> None:
"""Remove empty parent folders without crossing the configured root."""
root = root.resolve()
parent = file_path.parent
while parent != root:
try:
parent.rmdir()
except OSError:
break
parent = parent.parent
def _verified_stage_path(version) -> Path | None:
"""Resolve the staged path and reject traversal outside storage root."""
if not version or not getattr(version, "local_relative_path", None):
return None
root = DEFAULT_STORAGE_ROOT.resolve()
candidate = (DEFAULT_STORAGE_ROOT / Path(version.local_relative_path)).resolve()
try:
candidate.relative_to(root)
except ValueError:
return None
return candidate
def _cleanup_verified_vps_stage(*, job, version) -> dict[str, object]:
"""Delete one VPS stage only after verified branch-local storage."""
result: dict[str, object] = {"enabled": DELETE_VPS_AFTER_LOCAL_SYNC, "deleted": False}
if not DELETE_VPS_AFTER_LOCAL_SYNC:
result["reason"] = "cleanup_disabled"
return result
if not job or job.status != "completed":
result["reason"] = "job_not_completed"
return result
if getattr(job, "vps_cleanup_status", "pending") == "deleted":
result.update({"deleted": True, "already_completed": True, "reason": "already_deleted"})
return result
expected = (job.expected_hash_sha256 or "").strip().lower()
acknowledged = (job.acknowledged_hash_sha256 or "").strip().lower()
if not expected or acknowledged != expected:
result["reason"] = "local_hash_not_verified"
return result
if not (job.local_final_path or "").strip():
result["reason"] = "local_final_path_missing"
return result
job.vps_cleanup_attempts = int(job.vps_cleanup_attempts or 0) + 1
job.vps_cleanup_status = "deleting"
job.vps_cleanup_error = None
stage_path = _verified_stage_path(version)
if stage_path is None:
job.vps_cleanup_status = "failed"
job.vps_cleanup_error = "Unsafe or missing staged path."
result["reason"] = "unsafe_stage_path"
return result
try:
deleted_bytes = int(stage_path.stat().st_size) if stage_path.exists() and stage_path.is_file() else 0
if stage_path.exists():
if not stage_path.is_file():
raise OSError("Staged path is not a regular file.")
stage_path.unlink()
_prune_empty_stage_directories(stage_path, DEFAULT_STORAGE_ROOT)
job.vps_cleanup_status = "deleted"
job.vps_cleanup_error = None
job.vps_deleted_at_utc = _now_utc()
job.vps_deleted_bytes = deleted_bytes
result.update({
"deleted": True,
"already_missing": deleted_bytes == 0,
"deleted_bytes": deleted_bytes,
"reason": "deleted_after_verified_local_sync",
})
return result
except OSError as exc:
job.vps_cleanup_status = "retry" if int(job.vps_cleanup_attempts or 0) < 20 else "failed"
job.vps_cleanup_error = f"{type(exc).__name__}: {str(exc)[:900]}"
result.update({"reason": "delete_failed", "error": job.vps_cleanup_error})
return result
def cleanup_completed_storage_job_vps_stage(db: Session, job: DocumentStorageJob) -> dict[str, object]:
version = db.get(EngagementDocumentVersion, job.version_id) if job else None
return _cleanup_verified_vps_stage(job=job, version=version)
def cleanup_completed_permanent_storage_job_vps_stage(db: Session, job: PermanentDocumentStorageJob) -> dict[str, object]:
version = db.get(PermanentClientDocumentVersion, job.version_id) if job else None
return _cleanup_verified_vps_stage(job=job, version=version)
def retry_verified_vps_cleanup_for_node(db: Session, node: BranchStorageNode, limit: int = 50) -> dict[str, int]:
"""Retry interrupted/failed deletion during heartbeat or tunnel sync."""
summary = {"checked": 0, "deleted": 0, "failed": 0}
if not DELETE_VPS_AFTER_LOCAL_SYNC or not node:
return summary
retry_states = ["pending", "retry", "failed", "deleting"]
engagement_jobs = db.execute(
select(DocumentStorageJob)
.where(
DocumentStorageJob.storage_node_id == node.id,
DocumentStorageJob.status == "completed",
DocumentStorageJob.vps_cleanup_status.in_(retry_states),
)
.order_by(DocumentStorageJob.completed_at_utc.asc(), DocumentStorageJob.id.asc())
.limit(limit)
).scalars().all()
remaining = max(0, limit - len(engagement_jobs))
permanent_jobs = []
if remaining:
permanent_jobs = db.execute(
select(PermanentDocumentStorageJob)
.where(
PermanentDocumentStorageJob.storage_node_id == node.id,
PermanentDocumentStorageJob.status == "completed",
PermanentDocumentStorageJob.vps_cleanup_status.in_(retry_states),
)
.order_by(PermanentDocumentStorageJob.completed_at_utc.asc(), PermanentDocumentStorageJob.id.asc())
.limit(remaining)
).scalars().all()
for job in engagement_jobs:
summary["checked"] += 1
result = cleanup_completed_storage_job_vps_stage(db, job)
summary["deleted" if result.get("deleted") else "failed"] += 1
for job in permanent_jobs:
summary["checked"] += 1
result = cleanup_completed_permanent_storage_job_vps_stage(db, job)
summary["deleted" if result.get("deleted") else "failed"] += 1
return summary
def list_storage_jobs(db: Session, tenant_id: int | None = None, branch_id: int | None = None, status: str | None = None, limit: int = 200):
stmt = select(DocumentStorageJob).order_by(DocumentStorageJob.created_at_utc.desc())
if tenant_id:
app/modules/documents/ui.py
+17 -2
View File
@@ -27,6 +27,9 @@ from app.modules.documents.services import (
get_latest_version,
get_version,
acknowledge_storage_job,
cleanup_completed_storage_job_vps_stage,
cleanup_completed_permanent_storage_job_vps_stage,
retry_verified_vps_cleanup_for_node,
authenticate_storage_node,
create_branch_storage_node,
generate_storage_secret,
@@ -1568,8 +1571,9 @@ async def storage_agent_heartbeat(request: Request, x_node_code: str | None = He
node, error = _agent_auth(db, request, x_node_code, x_node_secret)
if error:
return error
cleanup = retry_verified_vps_cleanup_for_node(db, node, limit=50)
db.commit()
return {"ok": True, "node_code": node.node_code, "status": node.status}
return {"ok": True, "node_code": node.node_code, "status": node.status, "vps_cleanup": cleanup}
finally:
db.close()
@@ -1644,6 +1648,8 @@ def _permanent_download_requests_payload(items):
def _storage_agent_sync_payload(db, node):
# Retry cleanup in tunnel mode as well as normal heartbeat mode.
retry_verified_vps_cleanup_for_node(db, node, limit=25)
jobs = _normal_storage_jobs_payload(list_pending_storage_jobs(db, node))
jobs += _permanent_storage_jobs_payload(list_pending_permanent_storage_jobs(db, node))
requests_ = _normal_download_requests_payload(list_pending_download_requests(db, node))
@@ -1719,8 +1725,17 @@ async def storage_agent_ack_job(request: Request, job_id: str, x_node_code: str
if not job:
return JSONResponse({"ok": False, "error": "job_not_found"}, status_code=404)
ok = acknowledge_storage_job(db, node=node, job=job, acknowledged_hash=acknowledged_hash, local_final_path=local_final_path, success=bool(payload.get("success", True)), error=payload.get("error"))
# First commit the verified local acknowledgement. Only after that
# durable commit is the VPS staging file eligible for deletion.
db.commit()
return {"ok": ok, "job_status": job.status}
cleanup = {"enabled": True, "deleted": False, "reason": "acknowledgement_failed"}
if ok:
if is_permanent:
cleanup = cleanup_completed_permanent_storage_job_vps_stage(db, job)
else:
cleanup = cleanup_completed_storage_job_vps_stage(db, job)
db.commit()
return {"ok": ok, "job_status": job.status, "vps_cleanup": cleanup}
finally:
db.close()