Harden Local Agent singleton supervisor and worker lifecycle
This commit is contained in:
@@ -21,6 +21,7 @@ LOG_DIR = INSTALL_ROOT / "logs"
|
||||
UPDATES_DIR = INSTALL_ROOT / "updates"
|
||||
LOCK_FILE = DATA_DIR / "supervisor.lock"
|
||||
STATE_FILE = DATA_DIR / "supervisor_state.json"
|
||||
OWNER_FILE = DATA_DIR / "supervisor_owner.json"
|
||||
REQUEST_FILE = UPDATES_DIR / "supervisor_request.json"
|
||||
WORKER_LOG = LOG_DIR / "worker-supervisor.log"
|
||||
SUPERVISOR_LOG = LOG_DIR / "supervisor.log"
|
||||
@@ -29,12 +30,13 @@ WORKER_MODULE = "erp_local_agent.main"
|
||||
RESTART_DELAY_SECONDS = 3
|
||||
HEALTH_TIMEOUT_SECONDS = 60
|
||||
ERROR_ALREADY_EXISTS = 183
|
||||
SINGLETON_WATCHDOG_SECONDS = 10
|
||||
|
||||
|
||||
def _supervisor_mutex_name() -> str:
|
||||
normalized = str(INSTALL_ROOT.resolve()).casefold().encode("utf-8", errors="ignore")
|
||||
suffix = hashlib.sha256(normalized).hexdigest()[:20]
|
||||
return f"Local\\ARRR_ERP_Local_Agent_SUPERVISOR_{suffix}"
|
||||
return f"Global\\ARRR_ERP_Local_Agent_SUPERVISOR_{suffix}"
|
||||
|
||||
|
||||
def _acquire_os_mutex():
|
||||
@@ -188,6 +190,42 @@ def _kill_unmanaged_workers() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _enforce_singleton_processes(*, keep_worker_pid: int | None = None) -> None:
|
||||
"""Kill stale/duplicate supervisors and workers for this install only.
|
||||
|
||||
The Global named mutex is the primary atomic guard. This watchdog is a
|
||||
recovery layer for legacy processes started before the hardened runtime.
|
||||
"""
|
||||
if os.name != "nt":
|
||||
return
|
||||
root = str(INSTALL_ROOT.resolve()).replace("'", "''")
|
||||
keep_worker = int(keep_worker_pid or 0)
|
||||
script = rf"""
|
||||
$root='{root}'
|
||||
$selfPid={os.getpid()}
|
||||
$keepWorker={keep_worker}
|
||||
Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object {{
|
||||
$cmd=[string]$_.CommandLine
|
||||
if (-not $cmd -or $cmd -notlike ('*'+$root+'*')) {{ return }}
|
||||
if (($cmd -match 'ERPAgentSupervisor\.pyw') -and ($_.ProcessId -ne $selfPid)) {{
|
||||
Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue
|
||||
return
|
||||
}}
|
||||
if (($cmd -match 'erp_local_agent\.main') -and ($_.ProcessId -ne $keepWorker)) {{
|
||||
Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
subprocess.run(
|
||||
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", script],
|
||||
cwd=str(INSTALL_ROOT),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
creationflags=_create_no_window(),
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
class Supervisor:
|
||||
def __init__(self):
|
||||
self.worker: subprocess.Popen | None = None
|
||||
@@ -209,6 +247,11 @@ class Supervisor:
|
||||
if existing and existing != os.getpid() and _pid_alive(existing):
|
||||
return False
|
||||
LOCK_FILE.write_text(str(os.getpid()), encoding="utf-8")
|
||||
_write_json_atomic(OWNER_FILE, {
|
||||
"supervisor_pid": os.getpid(),
|
||||
"install_root": str(INSTALL_ROOT),
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
})
|
||||
return True
|
||||
|
||||
def release_lock(self) -> None:
|
||||
@@ -217,6 +260,13 @@ class Supervisor:
|
||||
LOCK_FILE.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if OWNER_FILE.exists():
|
||||
owner = json.loads(OWNER_FILE.read_text(encoding="utf-8"))
|
||||
if int(owner.get("supervisor_pid") or 0) == os.getpid():
|
||||
OWNER_FILE.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
_close_os_mutex(self._os_mutex)
|
||||
self._os_mutex = None
|
||||
|
||||
@@ -436,8 +486,16 @@ class Supervisor:
|
||||
_kill_duplicate_supervisors()
|
||||
_state("starting", "ERP Local Agent supervisor starting.")
|
||||
self.start_worker()
|
||||
last_singleton_watchdog = 0.0
|
||||
|
||||
while self.running:
|
||||
now = time.time()
|
||||
if now - last_singleton_watchdog >= SINGLETON_WATCHDOG_SECONDS:
|
||||
_enforce_singleton_processes(
|
||||
keep_worker_pid=(self.worker.pid if self.worker is not None and self.worker.poll() is None else None)
|
||||
)
|
||||
last_singleton_watchdog = now
|
||||
|
||||
request = self._read_request()
|
||||
if request and str(request.get("action") or "") == "install_update":
|
||||
self._clear_request()
|
||||
@@ -464,7 +522,17 @@ class Supervisor:
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="ERP Local Agent desktop/background supervisor")
|
||||
parser.add_argument("--background", action="store_true")
|
||||
parser.parse_args()
|
||||
parser.add_argument("--replace", action="store_true", help="Explicit controlled replacement of an older supervisor instance")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.replace and os.name == "nt":
|
||||
# Explicit restart/update only. Normal duplicate launches never disturb
|
||||
# the healthy owner; they simply fail the Global mutex and exit.
|
||||
_kill_duplicate_supervisors()
|
||||
deadline = time.time() + 15
|
||||
while time.time() < deadline:
|
||||
time.sleep(0.25)
|
||||
break
|
||||
|
||||
supervisor = Supervisor()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user