Harden Local Agent singleton supervisor and worker lifecycle
This commit is contained in:
@@ -4,7 +4,7 @@ import io
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
ERP_LOCAL_AGENT_VERSION = "1.22.10"
|
ERP_LOCAL_AGENT_VERSION = "1.22.11"
|
||||||
ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
|
ERP_LOCAL_AGENT_NAME = "ERP Local Agent"
|
||||||
RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
|
RUNTIME_ROOT = Path(__file__).resolve().parent / "local_agent_runtime"
|
||||||
_DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
|
_DETERMINISTIC_ZIP_TIMESTAMP = (2026, 1, 1, 0, 0, 0)
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ ERROR_ALREADY_EXISTS = 183
|
|||||||
def _dashboard_mutex_name() -> str:
|
def _dashboard_mutex_name() -> str:
|
||||||
normalized = str(INSTALL_ROOT.resolve()).casefold().encode("utf-8", errors="ignore")
|
normalized = str(INSTALL_ROOT.resolve()).casefold().encode("utf-8", errors="ignore")
|
||||||
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
||||||
return f"Local\\ARRR_ERP_Local_Agent_DASHBOARD_{suffix}"
|
return f"Global\\ARRR_ERP_Local_Agent_DASHBOARD_{suffix}"
|
||||||
|
|
||||||
|
|
||||||
def _acquire_dashboard_mutex():
|
def _acquire_dashboard_mutex():
|
||||||
@@ -142,6 +142,26 @@ def _pythonw() -> Path:
|
|||||||
def _ensure_supervisor() -> None:
|
def _ensure_supervisor() -> None:
|
||||||
if _supervisor_running():
|
if _supervisor_running():
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# The Scheduled Task is the canonical owner/launcher. Starting it repeatedly
|
||||||
|
# is safe because the task is configured with MultipleInstances=IgnoreNew,
|
||||||
|
# and the supervisor itself also owns a machine-wide Global mutex.
|
||||||
|
started_by_task = False
|
||||||
|
if os.name == "nt":
|
||||||
|
result = subprocess.run(
|
||||||
|
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command",
|
||||||
|
"Start-ScheduledTask -TaskName 'ERP Local Agent' -ErrorAction Stop"],
|
||||||
|
cwd=str(INSTALL_ROOT),
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
creationflags=_create_no_window(),
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
started_by_task = result.returncode == 0
|
||||||
|
|
||||||
|
if not started_by_task:
|
||||||
|
# Recovery fallback for a workstation where the scheduled task was not
|
||||||
|
# installed yet. The Global mutex still prevents a second supervisor.
|
||||||
subprocess.Popen(
|
subprocess.Popen(
|
||||||
[str(_pythonw()), str(SUPERVISOR_SCRIPT), "--background"],
|
[str(_pythonw()), str(SUPERVISOR_SCRIPT), "--background"],
|
||||||
cwd=str(INSTALL_ROOT),
|
cwd=str(INSTALL_ROOT),
|
||||||
@@ -150,6 +170,7 @@ def _ensure_supervisor() -> None:
|
|||||||
creationflags=_create_no_window(),
|
creationflags=_create_no_window(),
|
||||||
close_fds=True,
|
close_fds=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
deadline = time.time() + 20
|
deadline = time.time() + 20
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if _supervisor_running():
|
if _supervisor_running():
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ LOG_DIR = INSTALL_ROOT / "logs"
|
|||||||
UPDATES_DIR = INSTALL_ROOT / "updates"
|
UPDATES_DIR = INSTALL_ROOT / "updates"
|
||||||
LOCK_FILE = DATA_DIR / "supervisor.lock"
|
LOCK_FILE = DATA_DIR / "supervisor.lock"
|
||||||
STATE_FILE = DATA_DIR / "supervisor_state.json"
|
STATE_FILE = DATA_DIR / "supervisor_state.json"
|
||||||
|
OWNER_FILE = DATA_DIR / "supervisor_owner.json"
|
||||||
REQUEST_FILE = UPDATES_DIR / "supervisor_request.json"
|
REQUEST_FILE = UPDATES_DIR / "supervisor_request.json"
|
||||||
WORKER_LOG = LOG_DIR / "worker-supervisor.log"
|
WORKER_LOG = LOG_DIR / "worker-supervisor.log"
|
||||||
SUPERVISOR_LOG = LOG_DIR / "supervisor.log"
|
SUPERVISOR_LOG = LOG_DIR / "supervisor.log"
|
||||||
@@ -29,12 +30,13 @@ WORKER_MODULE = "erp_local_agent.main"
|
|||||||
RESTART_DELAY_SECONDS = 3
|
RESTART_DELAY_SECONDS = 3
|
||||||
HEALTH_TIMEOUT_SECONDS = 60
|
HEALTH_TIMEOUT_SECONDS = 60
|
||||||
ERROR_ALREADY_EXISTS = 183
|
ERROR_ALREADY_EXISTS = 183
|
||||||
|
SINGLETON_WATCHDOG_SECONDS = 10
|
||||||
|
|
||||||
|
|
||||||
def _supervisor_mutex_name() -> str:
|
def _supervisor_mutex_name() -> str:
|
||||||
normalized = str(INSTALL_ROOT.resolve()).casefold().encode("utf-8", errors="ignore")
|
normalized = str(INSTALL_ROOT.resolve()).casefold().encode("utf-8", errors="ignore")
|
||||||
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
||||||
return f"Local\\ARRR_ERP_Local_Agent_SUPERVISOR_{suffix}"
|
return f"Global\\ARRR_ERP_Local_Agent_SUPERVISOR_{suffix}"
|
||||||
|
|
||||||
|
|
||||||
def _acquire_os_mutex():
|
def _acquire_os_mutex():
|
||||||
@@ -188,6 +190,42 @@ def _kill_unmanaged_workers() -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _enforce_singleton_processes(*, keep_worker_pid: int | None = None) -> None:
|
||||||
|
"""Kill stale/duplicate supervisors and workers for this install only.
|
||||||
|
|
||||||
|
The Global named mutex is the primary atomic guard. This watchdog is a
|
||||||
|
recovery layer for legacy processes started before the hardened runtime.
|
||||||
|
"""
|
||||||
|
if os.name != "nt":
|
||||||
|
return
|
||||||
|
root = str(INSTALL_ROOT.resolve()).replace("'", "''")
|
||||||
|
keep_worker = int(keep_worker_pid or 0)
|
||||||
|
script = rf"""
|
||||||
|
$root='{root}'
|
||||||
|
$selfPid={os.getpid()}
|
||||||
|
$keepWorker={keep_worker}
|
||||||
|
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object {{
|
||||||
|
$cmd=[string]$_.CommandLine
|
||||||
|
if (-not $cmd -or $cmd -notlike ('*'+$root+'*')) {{ return }}
|
||||||
|
if (($cmd -match 'ERPAgentSupervisor\.pyw') -and ($_.ProcessId -ne $selfPid)) {{
|
||||||
|
Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue
|
||||||
|
return
|
||||||
|
}}
|
||||||
|
if (($cmd -match 'erp_local_agent\.main') -and ($_.ProcessId -ne $keepWorker)) {{
|
||||||
|
Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
"""
|
||||||
|
subprocess.run(
|
||||||
|
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", script],
|
||||||
|
cwd=str(INSTALL_ROOT),
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
creationflags=_create_no_window(),
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Supervisor:
|
class Supervisor:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.worker: subprocess.Popen | None = None
|
self.worker: subprocess.Popen | None = None
|
||||||
@@ -209,6 +247,11 @@ class Supervisor:
|
|||||||
if existing and existing != os.getpid() and _pid_alive(existing):
|
if existing and existing != os.getpid() and _pid_alive(existing):
|
||||||
return False
|
return False
|
||||||
LOCK_FILE.write_text(str(os.getpid()), encoding="utf-8")
|
LOCK_FILE.write_text(str(os.getpid()), encoding="utf-8")
|
||||||
|
_write_json_atomic(OWNER_FILE, {
|
||||||
|
"supervisor_pid": os.getpid(),
|
||||||
|
"install_root": str(INSTALL_ROOT),
|
||||||
|
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||||
|
})
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def release_lock(self) -> None:
|
def release_lock(self) -> None:
|
||||||
@@ -217,6 +260,13 @@ class Supervisor:
|
|||||||
LOCK_FILE.unlink()
|
LOCK_FILE.unlink()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
try:
|
||||||
|
if OWNER_FILE.exists():
|
||||||
|
owner = json.loads(OWNER_FILE.read_text(encoding="utf-8"))
|
||||||
|
if int(owner.get("supervisor_pid") or 0) == os.getpid():
|
||||||
|
OWNER_FILE.unlink()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
_close_os_mutex(self._os_mutex)
|
_close_os_mutex(self._os_mutex)
|
||||||
self._os_mutex = None
|
self._os_mutex = None
|
||||||
|
|
||||||
@@ -436,8 +486,16 @@ class Supervisor:
|
|||||||
_kill_duplicate_supervisors()
|
_kill_duplicate_supervisors()
|
||||||
_state("starting", "ERP Local Agent supervisor starting.")
|
_state("starting", "ERP Local Agent supervisor starting.")
|
||||||
self.start_worker()
|
self.start_worker()
|
||||||
|
last_singleton_watchdog = 0.0
|
||||||
|
|
||||||
while self.running:
|
while self.running:
|
||||||
|
now = time.time()
|
||||||
|
if now - last_singleton_watchdog >= SINGLETON_WATCHDOG_SECONDS:
|
||||||
|
_enforce_singleton_processes(
|
||||||
|
keep_worker_pid=(self.worker.pid if self.worker is not None and self.worker.poll() is None else None)
|
||||||
|
)
|
||||||
|
last_singleton_watchdog = now
|
||||||
|
|
||||||
request = self._read_request()
|
request = self._read_request()
|
||||||
if request and str(request.get("action") or "") == "install_update":
|
if request and str(request.get("action") or "") == "install_update":
|
||||||
self._clear_request()
|
self._clear_request()
|
||||||
@@ -464,7 +522,17 @@ class Supervisor:
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description="ERP Local Agent desktop/background supervisor")
|
parser = argparse.ArgumentParser(description="ERP Local Agent desktop/background supervisor")
|
||||||
parser.add_argument("--background", action="store_true")
|
parser.add_argument("--background", action="store_true")
|
||||||
parser.parse_args()
|
parser.add_argument("--replace", action="store_true", help="Explicit controlled replacement of an older supervisor instance")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.replace and os.name == "nt":
|
||||||
|
# Explicit restart/update only. Normal duplicate launches never disturb
|
||||||
|
# the healthy owner; they simply fail the Global mutex and exit.
|
||||||
|
_kill_duplicate_supervisors()
|
||||||
|
deadline = time.time() + 15
|
||||||
|
while time.time() < deadline:
|
||||||
|
time.sleep(0.25)
|
||||||
|
break
|
||||||
|
|
||||||
supervisor = Supervisor()
|
supervisor = Supervisor()
|
||||||
|
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
__version__ = "1.22.10"
|
__version__ = "1.22.11"
|
||||||
AGENT_NAME = "ERP Local Agent"
|
AGENT_NAME = "ERP Local Agent"
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ ERROR_ALREADY_EXISTS = 183
|
|||||||
def _mutex_name(scope: str, root: Path) -> str:
|
def _mutex_name(scope: str, root: Path) -> str:
|
||||||
normalized = str(root.resolve()).casefold().encode("utf-8", errors="ignore")
|
normalized = str(root.resolve()).casefold().encode("utf-8", errors="ignore")
|
||||||
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
||||||
return f"Local\\ARRR_ERP_Local_Agent_{scope}_{suffix}"
|
return f"Global\\ARRR_ERP_Local_Agent_{scope}_{suffix}"
|
||||||
|
|
||||||
|
|
||||||
def acquire_named_mutex(scope: str, root: Path):
|
def acquire_named_mutex(scope: str, root: Path):
|
||||||
|
|||||||
Reference in New Issue
Block a user