Improve Local Agent update reliability v1.26.24

2026-09-30 22:28:39 +05:30
parent 56098aaeb9
commit 24d7a8b1ec
6 changed files with 96 additions and 8 deletions
@@ -4,7 +4,7 @@ import io
 from pathlib import Path
 import zipfile

-ERP_LOCAL_AGENT_VERSION = "1.26.23"
+ERP_LOCAL_AGENT_VERSION = "1.26.24"
 ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
 RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
 _DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
@@ -23,6 +23,7 @@ STATE_FILE = DATA_DIR / "supervisor_state.json"
 OWNER_FILE = DATA_DIR / "supervisor_owner.json"
 REQUEST_FILE = UPDATES_DIR / "supervisor_request.json"
 UPDATE_PROGRESS_FILE = DATA_DIR / "update_progress.json"
+FAILED_UPDATES_FILE = DATA_DIR / "failed_updates.json"
 WORKER_LOG = LOG_DIR / "worker-supervisor.log"
 SUPERVISOR_LOG = LOG_DIR / "supervisor.log"
 DASHBOARD_URL = "http://127.0.0.1:8788"
@@ -110,6 +111,36 @@ def _update_progress(phase: str, percent: int, message: str, *, version: str = "
    _write_json_atomic(UPDATE_PROGRESS_FILE, payload)


+def _read_failed_updates() -> dict:
+    try:
+        if FAILED_UPDATES_FILE.exists():
+            value = json.loads(FAILED_UPDATES_FILE.read_text(encoding="utf-8"))
+            return value if isinstance(value, dict) else {}
+    except Exception:
+        pass
+    return {}
+
+
+def _mark_failed_update(version: str, error: str) -> None:
+    if not version:
+        return
+    payload = _read_failed_updates()
+    previous = payload.get(version) if isinstance(payload.get(version), dict) else {}
+    payload[version] = {
+        "failed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "error": str(error)[:2000],
+        "attempts": int(previous.get("attempts") or 0) + 1,
+    }
+    _write_json_atomic(FAILED_UPDATES_FILE, payload)
+
+
+def _clear_failed_update(version: str) -> None:
+    payload = _read_failed_updates()
+    if version in payload:
+        payload.pop(version, None)
+        _write_json_atomic(FAILED_UPDATES_FILE, payload)
+
+
 def _dashboard_ready(timeout: float = 1.5) -> bool:
    try:
        with urllib.request.urlopen(DASHBOARD_URL + "/api/status", timeout=timeout) as response:
@@ -443,6 +474,7 @@ class Supervisor:
                    raise RuntimeError(f"Updated worker exited with code {self.worker.returncode}.")
                if _dashboard_ready():
                    _update_progress("complete", 100, f"ERP Local Agent {version} updated successfully.", version=version, status="updated")
+                    _clear_failed_update(version)
                    _state(
                        "updated",
                        f"ERP Local Agent {version} installed and restarted successfully.",
@@ -456,6 +488,7 @@ class Supervisor:
                time.sleep(1)
            raise RuntimeError("Updated worker did not become healthy within the restart timeout.")
        except Exception as exc:
+            _mark_failed_update(version, str(exc))
            _update_progress("rollback", 92, f"Update failed; restoring the previous runtime: {exc}", version=version, status="rollback")
            _state("rollback", f"Update {version} failed. Restoring previous worker.", target_version=version)
            self.stop_worker()
@@ -1,2 +1,2 @@
-__version__ = "1.26.23"
+__version__ = "1.26.24"
 AGENT_NAME = "ERP Local Agent"
@@ -105,9 +105,17 @@ class ERPClient:
            response = self.session.post(self._url(f"/documents/storage-agent/download-requests/{request_id}/upload"), files=files, data=data, timeout=max(self.config.request_timeout_seconds, 300))
        response.raise_for_status(); return response.json() if response.content else {"status": "ok"}

-    def update_manifest(self) -> dict[str, Any]:
-        response = self.session.get(self._url("/documents/erp-local-agent/update-manifest"), timeout=self.config.request_timeout_seconds)
-        response.raise_for_status(); return response.json()
+    def update_manifest(self, *, timeout_seconds: int = 10) -> dict[str, Any]:
+        # Update checks are UI/background housekeeping and must never inherit the
+        # long general-purpose agent timeout. Keep connect/read bounds short so a
+        # slow or temporarily unreachable ERP cannot leave the dashboard at 10%.
+        timeout_seconds = max(3, min(int(timeout_seconds or 10), 15))
+        response = self.session.get(
+            self._url("/documents/erp-local-agent/update-manifest"),
+            timeout=(min(5, timeout_seconds), timeout_seconds),
+        )
+        response.raise_for_status()
+        return response.json()

    def download_update_package(self) -> bytes:
        response = self.session.get(self._url("/documents/erp-local-agent/update-package"), timeout=max(self.config.request_timeout_seconds, 300))
@@ -163,7 +163,7 @@ class AgentDashboard:
                        result=start_operator_download(token,str(dashboard.config.erp_base_url or ""),data_root)
                        return self._json({"ok":True,**result})
                    if self.path == "/api/update/check":
-                        return self._json({"ok": True, "update": dashboard.updater.check_for_update(force=True)})
+                        return self._json({"ok": True, "update": dashboard.updater.start_check()})
                    if self.path == "/api/update/download":
                        return self._json({"ok": True, "update": dashboard.updater.start_download()})
                    if self.path == "/api/update/install":
@@ -289,7 +289,7 @@ if(installWatching&&installTarget&&s.agent.version===installTarget){setProgress(
 await refreshHistory();await refreshMirrorProgress();}catch(e){if(installWatching){offlineTicks++;let estimated=Math.min(92,Math.max(lastServerPercent,35+offlineTicks*3));setProgress('Applying Update',estimated,'Local Agent is restarting. Waiting for the updated dashboard to come back online…',true)}else{console.log(e)}}}
 async function refreshHistory(){try{const r=await fetch('/api/history',{cache:'no-store'});const h=await r.json();let rows=h.events.slice(0,12).map(x=>'<tr><td>'+x.occurred_at_utc+'</td><td>'+x.level+'</td><td>'+x.event_type+'</td><td>'+escapeHtml(x.message)+'</td></tr>').join('');document.getElementById('history').innerHTML='<table><thead><tr><th>Time UTC</th><th>Level</th><th>Event</th><th>Message</th></tr></thead><tbody>'+rows+'</tbody></table>'}catch(e){}}
 function escapeHtml(v){return String(v||'').replace(/[&<>'"]/g,m=>({'&':'&amp;','<':'&lt;','>':'&gt;',"'":'&#039;','"':'&quot;'}[m]))}
-async function checkUpdate(){setProgress('Checking for Updates',10,'Checking ERP for the latest Local Agent version…',true);msg('Checking for updates…',true);try{const r=await fetch('/api/update/check',{method:'POST'});const j=await r.json();if(!r.ok||!j.ok)throw new Error(j.error||'Update check failed');applyProgress(j.update);msg(j.update.update_available?'Update '+j.update.latest_version+' is available.':'Local Agent is up to date.',true);await refresh()}catch(e){setProgress('Update Check',100,e.message,false);msg(e.message,false)}}
+async function checkUpdate(){setProgress('Checking for Updates',10,'Checking ERP for the latest Local Agent version…',true);msg('Checking for updates…',true);try{const r=await fetch('/api/update/check',{method:'POST'});const j=await r.json();if(!r.ok||!j.ok)throw new Error(j.error||'Update check failed');applyProgress(j.update);msg('Update check started. This page will update automatically.',true);await refresh()}catch(e){setProgress('Update Check',100,e.message,false);msg(e.message,false)}}
 async function downloadUpdate(){setProgress('Downloading Update',5,'Starting download…',true);msg('Downloading update…',true);try{const r=await fetch('/api/update/download',{method:'POST'});const j=await r.json();if(!r.ok||!j.ok)throw new Error(j.error||'Download failed');msg('Download started. You can keep this dashboard open.',true);await refresh()}catch(e){setProgress('Download Update',100,e.message,false);msg(e.message,false)}}
 async function action(url){msg('Working…',true);try{const r=await fetch(url,{method:'POST'});const j=await r.json();if(!r.ok||!j.ok)throw new Error(j.error||'Operation failed');msg('Completed successfully.',true);await refresh()}catch(e){msg(e.message,false)}}
 async function installUpdate(){if(!confirm('Install the downloaded update now? The Local Agent worker will restart safely and this dashboard will show progress until the updated dashboard returns.'))return;let latest=document.getElementById('latest').textContent.replace('Latest: ','').split(' •')[0];installTarget=latest;installWatching=true;offlineTicks=0;lastServerPercent=5;setProgress('Applying Update',5,'Queuing update with the Local Agent supervisor…',true);msg('Starting safe update installation…',true);try{const r=await fetch('/api/update/install',{method:'POST'});const j=await r.json();if(!r.ok||!j.ok)throw new Error(j.error||'Install failed');if(j.target_version)installTarget=j.target_version;msg(j.message||'Update queued. Keep this dashboard open.',true);setProgress('Applying Update',8,'Update queued. Waiting for supervisor…',true)}catch(e){installWatching=false;setProgress('Apply Update',100,e.message,false);msg(e.message,false)}}
@@ -86,6 +86,20 @@ class AgentUpdater:
            pass
        return {}

+
+    def _failed_update_versions(self) -> dict:
+        path = self.install_dir / "data" / "failed_updates.json"
+        try:
+            if path.exists():
+                value = json.loads(path.read_text(encoding="utf-8"))
+                return value if isinstance(value, dict) else {}
+        except Exception:
+            self.logger.exception("Could not read failed update quarantine")
+        return {}
+
+    def _is_quarantined(self, version: str) -> bool:
+        return bool(version and version in self._failed_update_versions())
+
    def check_for_update(self, force: bool = False) -> dict:
        with self.lock:
            now = time.time()
@@ -94,7 +108,7 @@ class AgentUpdater:
            self.last_check = now
            self._set_progress("check", "checking", "checking", 10, "Checking ERP for Local Agent updates.")
            try:
-                manifest = self.client.update_manifest()
+                manifest = self.client.update_manifest(timeout_seconds=10)
                latest = str(manifest.get("latest_version") or "").strip()
                expected = str(manifest.get("sha256") or "").lower().strip()
                self.latest_manifest = dict(manifest or {})
@@ -115,6 +129,27 @@ class AgentUpdater:
                self.logger.exception("ERP Local Agent update check failed: %s", exc)
                raise

+    def start_check(self) -> dict:
+        """Start a manual update check without blocking the dashboard HTTP request."""
+        with self.lock:
+            if self.busy:
+                raise RuntimeError("Another update operation is already in progress.")
+            self.busy = True
+            self._set_progress("check", "checking", "checking", 10, "Checking ERP for Local Agent updates.")
+
+        def runner():
+            try:
+                self.check_for_update(force=True)
+            except Exception:
+                # check_for_update already records a user-visible terminal error.
+                pass
+            finally:
+                with self.lock:
+                    self.busy = False
+
+        threading.Thread(target=runner, name="erp-local-agent-update-check", daemon=True).start()
+        return self.status()
+
    def maybe_check(self, force: bool = False) -> bool:
        if not self.config.auto_update:
            return False
@@ -126,6 +161,17 @@ class AgentUpdater:
            if not self.config.auto_install_updates:
                return True
            latest = str(state.get("latest_version") or "").strip()
+            if self._is_quarantined(latest):
+                self.logger.warning(
+                    "ERP Local Agent %s previously failed installation; automatic retry is suppressed until a newer version is published or a user explicitly retries.",
+                    latest,
+                )
+                self._set_progress(
+                    "check", "available", "complete", 100,
+                    f"ERP Local Agent {latest} is available, but automatic installation is paused because this version previously failed. Use Install & Restart Safely to retry manually.",
+                    target_version=latest,
+                )
+                return True
            request_path = self.install_dir / "updates" / "supervisor_request.json"
            if request_path.exists():
                return True
@@ -291,6 +337,7 @@ class AgentUpdater:
            "automatic_check_enabled": bool(self.config.auto_update),
            "automatic_install_enabled": bool(self.config.auto_update and self.config.auto_install_updates),
            "manual_update_only": bool(getattr(self.config, "manual_update_only", False)),
+            "quarantined": self._is_quarantined(latest),
            "supervisor": supervisor_state,
            "progress": self._progress(),
        }