Add ERP Local Agent secure self update framework
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
cd /d %~dp0
|
||||
if not exist .venv\Scripts\python.exe (
|
||||
py -3 -m venv .venv
|
||||
)
|
||||
call .venv\Scripts\activate.bat
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
python check_config.py
|
||||
pause
|
||||
@@ -0,0 +1,21 @@
|
||||
from pathlib import Path
|
||||
from audit_storage_agent.config import load_config
|
||||
from audit_storage_agent.client import ERPClient
|
||||
from audit_storage_agent.logger import setup_logger
|
||||
from audit_storage_agent.storage_identity import ensure_storage_identity
|
||||
|
||||
base_dir = Path(__file__).resolve().parent
|
||||
logger = setup_logger(base_dir)
|
||||
config = load_config(str(base_dir / ".env"))
|
||||
logger.info("Configuration loaded successfully")
|
||||
logger.info("ERP_BASE_URL=%s", config.erp_base_url)
|
||||
logger.info("NODE_CODE=%s", config.node_code)
|
||||
logger.info("STORAGE_ROOT=%s", config.storage_root)
|
||||
logger.info("TENANT_ID=%s", config.tenant_id or "-")
|
||||
logger.info("BRANCH_ID=%s", config.branch_id or "-")
|
||||
identity_path = ensure_storage_identity(config)
|
||||
logger.info("Storage identity file verified: %s", identity_path)
|
||||
client = ERPClient(config)
|
||||
response = client.heartbeat({"status": "CONFIG_TEST", "storage_root": str(config.storage_root), "agent_version": "0.2.0"})
|
||||
logger.info("ERP heartbeat test response: %s", response)
|
||||
print("Configuration and ERP connection test completed successfully.")
|
||||
@@ -0,0 +1,2 @@
|
||||
__version__ = "1.0.0"
|
||||
AGENT_NAME = "ERP Local Agent"
|
||||
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
import requests
|
||||
|
||||
from .config import AgentConfig
|
||||
|
||||
|
||||
class ERPClient:
|
||||
def __init__(self, config: AgentConfig):
|
||||
self.config = config
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(config.headers)
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
return f"{self.config.erp_base_url}{path}"
|
||||
|
||||
def heartbeat(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
response = self.session.post(
|
||||
self._url("/documents/storage-agent/heartbeat"),
|
||||
json=payload,
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json() if response.content else {"status": "ok"}
|
||||
|
||||
def pending_storage_jobs(self) -> list[dict[str, Any]]:
|
||||
response = self.session.get(
|
||||
self._url("/documents/storage-agent/jobs/pending"),
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return data.get("jobs", [])
|
||||
|
||||
def download_storage_job(self, job_id: int | str):
|
||||
response = self.session.get(
|
||||
self._url(f"/documents/storage-agent/jobs/{job_id}/download"),
|
||||
stream=True,
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
def acknowledge_storage_job(self, job_id: int | str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
response = self.session.post(
|
||||
self._url(f"/documents/storage-agent/jobs/{job_id}/ack"),
|
||||
json=payload,
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json() if response.content else {"status": "ok"}
|
||||
|
||||
def pending_download_requests(self) -> list[dict[str, Any]]:
|
||||
response = self.session.get(
|
||||
self._url("/documents/storage-agent/download-requests/pending"),
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return data.get("requests", [])
|
||||
|
||||
def upload_download_request_file(self, request_id: int | str, file_path: Path, extra: dict[str, Any]) -> dict[str, Any]:
|
||||
with file_path.open("rb") as handle:
|
||||
files = {"file": (file_path.name, handle, "application/octet-stream")}
|
||||
data = {key: str(value) for key, value in extra.items() if value is not None}
|
||||
response = self.session.post(
|
||||
self._url(f"/documents/storage-agent/download-requests/{request_id}/upload"),
|
||||
files=files,
|
||||
data=data,
|
||||
timeout=max(self.config.request_timeout_seconds, 300),
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json() if response.content else {"status": "ok"}
|
||||
|
||||
|
||||
def update_manifest(self) -> dict[str, Any]:
|
||||
response = self.session.get(
|
||||
self._url("/documents/erp-local-agent/update-manifest"),
|
||||
timeout=self.config.request_timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def download_update_package(self) -> bytes:
|
||||
response = self.session.get(
|
||||
self._url("/documents/erp-local-agent/update-package"),
|
||||
timeout=max(self.config.request_timeout_seconds, 300),
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
@@ -0,0 +1,112 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlencode
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from . import __version__
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AgentConfig:
|
||||
erp_base_url: str
|
||||
node_code: str
|
||||
node_secret: str
|
||||
storage_root: Path
|
||||
tenant_id: str | None = None
|
||||
branch_id: str | None = None
|
||||
poll_interval_seconds: int = 30
|
||||
download_request_interval_seconds: int = 30
|
||||
heartbeat_interval_seconds: int = 60
|
||||
request_timeout_seconds: int = 60
|
||||
max_retries: int = 3
|
||||
tunnel_enabled: bool = True
|
||||
tunnel_reconnect_seconds: int = 10
|
||||
auto_update: bool = True
|
||||
update_check_interval_seconds: int = 300
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"X-Node-Code": self.node_code,
|
||||
"X-Node-Secret": self.node_secret,
|
||||
"User-Agent": f"ERPLocalAgent/{__version__}",
|
||||
}
|
||||
|
||||
@property
|
||||
def tunnel_url(self) -> str:
|
||||
base = self.erp_base_url.rstrip("/")
|
||||
if base.startswith("https://"):
|
||||
ws_base = "wss://" + base[len("https://"):]
|
||||
elif base.startswith("http://"):
|
||||
ws_base = "ws://" + base[len("http://"):]
|
||||
else:
|
||||
ws_base = base
|
||||
query = urlencode({"node_code": self.node_code, "node_secret": self.node_secret})
|
||||
return f"{ws_base}/documents/storage-agent/tunnel?{query}"
|
||||
|
||||
|
||||
def _get_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return default
|
||||
return str(raw).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def _get_int(name: str, default: int) -> int:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return default
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"Invalid integer for {name}: {raw!r}") from exc
|
||||
|
||||
|
||||
def load_config(env_file: str | None = None) -> AgentConfig:
|
||||
if env_file:
|
||||
load_dotenv(env_file)
|
||||
else:
|
||||
load_dotenv()
|
||||
|
||||
erp_base_url = os.getenv("ERP_BASE_URL", "").rstrip("/")
|
||||
node_code = os.getenv("NODE_CODE", "").strip()
|
||||
node_secret = os.getenv("NODE_SECRET", "").strip()
|
||||
storage_root_raw = os.getenv("STORAGE_ROOT", "").strip()
|
||||
tenant_id = os.getenv("TENANT_ID", os.getenv("AUDIT_FIRM_ID", "")).strip() or None
|
||||
branch_id = os.getenv("BRANCH_ID", "").strip() or None
|
||||
|
||||
missing = []
|
||||
if not erp_base_url:
|
||||
missing.append("ERP_BASE_URL")
|
||||
if not node_code:
|
||||
missing.append("NODE_CODE")
|
||||
if not node_secret:
|
||||
missing.append("NODE_SECRET")
|
||||
if not storage_root_raw:
|
||||
missing.append("STORAGE_ROOT")
|
||||
if missing:
|
||||
raise RuntimeError("Missing required configuration: " + ", ".join(missing))
|
||||
|
||||
storage_root = Path(storage_root_raw).expanduser().resolve()
|
||||
storage_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
return AgentConfig(
|
||||
erp_base_url=erp_base_url,
|
||||
node_code=node_code,
|
||||
node_secret=node_secret,
|
||||
storage_root=storage_root,
|
||||
tenant_id=tenant_id,
|
||||
branch_id=branch_id,
|
||||
poll_interval_seconds=_get_int("POLL_INTERVAL_SECONDS", 30),
|
||||
download_request_interval_seconds=_get_int("DOWNLOAD_REQUEST_INTERVAL_SECONDS", 30),
|
||||
heartbeat_interval_seconds=_get_int("HEARTBEAT_INTERVAL_SECONDS", 60),
|
||||
request_timeout_seconds=_get_int("REQUEST_TIMEOUT_SECONDS", 60),
|
||||
max_retries=_get_int("MAX_RETRIES", 3),
|
||||
tunnel_enabled=_get_bool("TUNNEL_ENABLED", True),
|
||||
tunnel_reconnect_seconds=_get_int("TUNNEL_RECONNECT_SECONDS", 10),
|
||||
auto_update=_get_bool("AUTO_UPDATE", True),
|
||||
update_check_interval_seconds=max(60, _get_int("UPDATE_CHECK_INTERVAL_SECONDS", 300)),
|
||||
)
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
class LocalDB:
|
||||
def __init__(self, db_path: Path):
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.db_path = db_path
|
||||
self._init()
|
||||
|
||||
def connect(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def _init(self) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS processed_storage_jobs (
|
||||
job_id TEXT PRIMARY KEY,
|
||||
local_path TEXT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
file_size INTEGER NOT NULL,
|
||||
processed_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS processed_download_requests (
|
||||
request_id TEXT PRIMARY KEY,
|
||||
local_path TEXT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
file_size INTEGER NOT NULL,
|
||||
processed_at TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def record_storage_job(self, job_id: str, local_path: str, sha256: str, file_size: int) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO processed_storage_jobs
|
||||
(job_id, local_path, sha256, file_size, processed_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(str(job_id), local_path, sha256, int(file_size), datetime.now(timezone.utc).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def record_download_request(self, request_id: str, local_path: str, sha256: str, file_size: int) -> None:
|
||||
with self.connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO processed_download_requests
|
||||
(request_id, local_path, sha256, file_size, processed_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(str(request_id), local_path, sha256, int(file_size), datetime.now(timezone.utc).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def setup_logger(base_dir: Path | None = None) -> logging.Logger:
|
||||
root = base_dir or Path.cwd()
|
||||
logs_dir = root / "logs"
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger("audit_storage_agent")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.handlers.clear()
|
||||
|
||||
formatter = logging.Formatter("%(asctime)s | %(levelname)s | %(name)s | %(message)s")
|
||||
|
||||
console = logging.StreamHandler()
|
||||
console.setFormatter(formatter)
|
||||
logger.addHandler(console)
|
||||
|
||||
file_handler = RotatingFileHandler(logs_dir / "agent.log", maxBytes=2_000_000, backupCount=5, encoding="utf-8")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .client import ERPClient
|
||||
from .config import load_config
|
||||
from .db import LocalDB
|
||||
from .logger import setup_logger
|
||||
from .sync import StorageAgent
|
||||
from .tunnel import StorageAgentTunnel
|
||||
from .updater import AgentUpdater
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="ERP Local Agent - storage and local ERP tool runtime")
|
||||
parser.add_argument("--env", default=None)
|
||||
parser.add_argument("--once", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
root = Path.cwd()
|
||||
logger = setup_logger(root)
|
||||
try:
|
||||
config = load_config(args.env)
|
||||
db = LocalDB(root / "data" / "agent.db")
|
||||
client = ERPClient(config)
|
||||
agent = StorageAgent(config, client, db, logger)
|
||||
updater = AgentUpdater(config, client, logger, root)
|
||||
if args.once:
|
||||
updater.maybe_update(force=True)
|
||||
agent.run_once()
|
||||
return 0
|
||||
def update_worker():
|
||||
while True:
|
||||
updater.maybe_update()
|
||||
time.sleep(max(60, config.update_check_interval_seconds))
|
||||
threading.Thread(target=update_worker, name="erp-local-agent-updater", daemon=True).start()
|
||||
if config.tunnel_enabled:
|
||||
asyncio.run(StorageAgentTunnel(agent).run_forever())
|
||||
else:
|
||||
agent.run_forever()
|
||||
return 0
|
||||
except KeyboardInterrupt:
|
||||
logger.info("ERP Local Agent stopped by user")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
logger.exception("ERP Local Agent failed: %s", exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
|
||||
_SAFE_SEGMENT = re.compile(r"[^A-Za-z0-9._ -]+")
|
||||
|
||||
|
||||
def sanitize_segment(value: str, fallback: str = "item") -> str:
|
||||
value = (value or "").strip().replace("/", "_").replace("\\", "_")
|
||||
value = _SAFE_SEGMENT.sub("_", value)
|
||||
value = value.strip(" ._")
|
||||
return value or fallback
|
||||
|
||||
|
||||
def safe_relative_path(raw_path: str | None, fallback_filename: str) -> Path:
|
||||
"""Return a safe relative path, preventing path traversal outside storage root."""
|
||||
if raw_path:
|
||||
parts = []
|
||||
for part in Path(str(raw_path).replace("\\", "/")).parts:
|
||||
if part in ("", ".", ".."):
|
||||
continue
|
||||
parts.append(sanitize_segment(part))
|
||||
if parts:
|
||||
return Path(*parts)
|
||||
return Path(sanitize_segment(fallback_filename, "document.bin"))
|
||||
|
||||
|
||||
def resolve_under_root(storage_root: Path, relative_path: Path) -> Path:
|
||||
target = (storage_root / relative_path).resolve()
|
||||
root = storage_root.resolve()
|
||||
if os.path.commonpath([str(root), str(target)]) != str(root):
|
||||
raise RuntimeError("Refusing to write/read outside storage root")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
return target
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_stream_to_file(stream: BinaryIO, destination: Path) -> tuple[int, str]:
|
||||
digest = hashlib.sha256()
|
||||
total = 0
|
||||
tmp = destination.with_suffix(destination.suffix + ".part")
|
||||
with tmp.open("wb") as out:
|
||||
while True:
|
||||
chunk = stream.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
digest.update(chunk)
|
||||
total += len(chunk)
|
||||
tmp.replace(destination)
|
||||
return total, digest.hexdigest()
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
IDENTITY_FILENAME = ".audit_storage_node.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StorageIdentity:
|
||||
erp_base_url: str
|
||||
node_code: str
|
||||
tenant_id: str | None
|
||||
branch_id: str | None
|
||||
storage_root: str
|
||||
agent_identity_version: int = 1
|
||||
|
||||
|
||||
def _normalise(value: Any) -> str:
|
||||
return str(value or "").strip().rstrip("/")
|
||||
|
||||
|
||||
def _normalise_optional(value: Any) -> str | None:
|
||||
value = str(value or "").strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def identity_path(storage_root: Path) -> Path:
|
||||
return storage_root / IDENTITY_FILENAME
|
||||
|
||||
|
||||
def build_identity(config) -> StorageIdentity:
|
||||
return StorageIdentity(
|
||||
erp_base_url=_normalise(config.erp_base_url),
|
||||
node_code=_normalise(config.node_code),
|
||||
tenant_id=_normalise_optional(getattr(config, "tenant_id", None)),
|
||||
branch_id=_normalise_optional(getattr(config, "branch_id", None)),
|
||||
storage_root=str(config.storage_root.resolve()),
|
||||
)
|
||||
|
||||
|
||||
def ensure_storage_identity(config, *, allow_initialise: bool = True) -> Path:
|
||||
"""Create/validate one identity file in the selected storage root.
|
||||
|
||||
This protects the branch storage folder from accidentally being reused by a
|
||||
different ERP node/branch, and gives the branch one stable storage root.
|
||||
It does not scan the whole machine; ERP still remains the final authority
|
||||
for enforcing one active node and one configured root per branch.
|
||||
"""
|
||||
root = config.storage_root.resolve()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
path = identity_path(root)
|
||||
expected = build_identity(config)
|
||||
|
||||
if not path.exists():
|
||||
if not allow_initialise:
|
||||
raise RuntimeError(
|
||||
f"Storage identity file is missing at {path}. Initialise the branch storage root first."
|
||||
)
|
||||
path.write_text(json.dumps(asdict(expected), indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
try:
|
||||
existing = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Storage identity file is unreadable/corrupt: {path}") from exc
|
||||
|
||||
checks = {
|
||||
"erp_base_url": expected.erp_base_url,
|
||||
"node_code": expected.node_code,
|
||||
"tenant_id": expected.tenant_id,
|
||||
"branch_id": expected.branch_id,
|
||||
}
|
||||
conflicts: list[str] = []
|
||||
for key, expected_value in checks.items():
|
||||
existing_value = existing.get(key)
|
||||
# Older packages may not have tenant_id/branch_id. Allow blank existing
|
||||
# values to be upgraded below, but never allow contradictory values.
|
||||
if key in {"tenant_id", "branch_id"} and not existing_value:
|
||||
continue
|
||||
if str(existing_value or "").strip().rstrip("/") != str(expected_value or "").strip().rstrip("/"):
|
||||
conflicts.append(f"{key}: existing={existing_value!r}, expected={expected_value!r}")
|
||||
|
||||
if conflicts:
|
||||
raise RuntimeError(
|
||||
"This storage folder is already linked to a different ERP storage node/branch. "
|
||||
"Use the existing branch storage package or reset/change the storage root from ERP. "
|
||||
+ "; ".join(conflicts)
|
||||
)
|
||||
|
||||
# Backfill missing tenant/branch fields and refresh resolved root path.
|
||||
changed = False
|
||||
updated = dict(existing)
|
||||
for key, value in asdict(expected).items():
|
||||
if updated.get(key) in (None, "") and value not in (None, ""):
|
||||
updated[key] = value
|
||||
changed = True
|
||||
if updated.get("storage_root") != expected.storage_root:
|
||||
updated["storage_root"] = expected.storage_root
|
||||
changed = True
|
||||
if changed:
|
||||
path.write_text(json.dumps(updated, indent=2), encoding="utf-8")
|
||||
return path
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .client import ERPClient
|
||||
from .config import AgentConfig
|
||||
from .db import LocalDB
|
||||
from .storage import resolve_under_root, safe_relative_path, sha256_file, sanitize_segment, write_stream_to_file
|
||||
from .storage_identity import ensure_storage_identity
|
||||
from . import __version__
|
||||
|
||||
|
||||
class StorageAgent:
|
||||
def __init__(self, config: AgentConfig, client: ERPClient, db: LocalDB, logger):
|
||||
self.config = config
|
||||
self.client = client
|
||||
self.db = db
|
||||
self.logger = logger
|
||||
self.last_heartbeat = 0.0
|
||||
self.identity_file = ensure_storage_identity(config)
|
||||
self.logger.info("Storage root identity verified: %s", self.identity_file)
|
||||
|
||||
def run_forever(self) -> None:
|
||||
self.logger.info("Starting ERP Local Agent for node=%s", self.config.node_code)
|
||||
while True:
|
||||
self.run_once()
|
||||
time.sleep(max(5, self.config.poll_interval_seconds))
|
||||
|
||||
def run_once(self) -> None:
|
||||
self._maybe_heartbeat()
|
||||
self.process_storage_jobs()
|
||||
self.process_download_requests()
|
||||
|
||||
def _maybe_heartbeat(self, force: bool = False) -> None:
|
||||
now = time.time()
|
||||
if not force and now - self.last_heartbeat < self.config.heartbeat_interval_seconds:
|
||||
return
|
||||
try:
|
||||
total, used, free = shutil.disk_usage(self.config.storage_root)
|
||||
payload = {
|
||||
"status": "ONLINE",
|
||||
"storage_root": str(self.config.storage_root),
|
||||
"total_bytes": total,
|
||||
"used_bytes": used,
|
||||
"free_bytes": free,
|
||||
"agent_version": __version__,
|
||||
"agent_name": "ERP Local Agent",
|
||||
"capabilities": ["storage"],
|
||||
"agent_time_utc": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
self.client.heartbeat(payload)
|
||||
self.last_heartbeat = now
|
||||
self.logger.info("Heartbeat sent. free_gb=%.2f", free / (1024 ** 3))
|
||||
except Exception as exc:
|
||||
self.logger.exception("Heartbeat failed: %s", exc)
|
||||
|
||||
def process_storage_jobs(self) -> None:
|
||||
try:
|
||||
jobs = self.client.pending_storage_jobs()
|
||||
except Exception as exc:
|
||||
self.logger.exception("Could not fetch pending storage jobs: %s", exc)
|
||||
return
|
||||
|
||||
self.process_storage_jobs_from_payload(jobs)
|
||||
|
||||
def process_storage_jobs_from_payload(self, jobs: list[dict[str, Any]]) -> None:
|
||||
if not jobs:
|
||||
return
|
||||
self.logger.info("Found %s pending storage job(s)", len(jobs))
|
||||
for job in jobs:
|
||||
try:
|
||||
self._process_storage_job(job)
|
||||
except Exception as exc:
|
||||
self.logger.exception("Storage job failed job=%s error=%s", job.get("id") or job.get("job_id"), exc)
|
||||
|
||||
def _process_storage_job(self, job: dict[str, Any]) -> None:
|
||||
job_id = job.get("id") or job.get("job_id")
|
||||
if job_id is None:
|
||||
raise RuntimeError(f"Storage job missing id: {job}")
|
||||
|
||||
original_filename = job.get("original_filename") or job.get("filename") or f"job_{job_id}.bin"
|
||||
relative_path_raw = job.get("local_relative_path") or job.get("relative_path")
|
||||
relative_path = safe_relative_path(relative_path_raw, original_filename)
|
||||
destination = resolve_under_root(self.config.storage_root, relative_path)
|
||||
|
||||
self.logger.info("Downloading storage job=%s to %s", job_id, destination)
|
||||
response = self.client.download_storage_job(job_id)
|
||||
file_size, digest = write_stream_to_file(response.raw, destination)
|
||||
|
||||
payload = {
|
||||
"status": "COMPLETED",
|
||||
"sha256_hash": digest,
|
||||
"file_hash": digest,
|
||||
"file_size": file_size,
|
||||
"local_relative_path": str(relative_path).replace("\\", "/"),
|
||||
"stored_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
self.client.acknowledge_storage_job(job_id, payload)
|
||||
self.db.record_storage_job(str(job_id), str(relative_path).replace("\\", "/"), digest, file_size)
|
||||
self.logger.info("Storage job completed job=%s size=%s sha256=%s", job_id, file_size, digest)
|
||||
|
||||
def process_download_requests(self) -> None:
|
||||
try:
|
||||
requests_ = self.client.pending_download_requests()
|
||||
except Exception as exc:
|
||||
self.logger.exception("Could not fetch pending download requests: %s", exc)
|
||||
return
|
||||
|
||||
self.process_download_requests_from_payload(requests_)
|
||||
|
||||
def process_download_requests_from_payload(self, requests_: list[dict[str, Any]]) -> None:
|
||||
if not requests_:
|
||||
return
|
||||
self.logger.info("Found %s pending download request(s)", len(requests_))
|
||||
for request in requests_:
|
||||
try:
|
||||
self._process_download_request(request)
|
||||
except Exception as exc:
|
||||
self.logger.exception("Download request failed request=%s error=%s", request.get("id") or request.get("request_id"), exc)
|
||||
|
||||
def _process_download_request(self, request: dict[str, Any]) -> None:
|
||||
request_id = request.get("id") or request.get("request_id")
|
||||
if request_id is None:
|
||||
raise RuntimeError(f"Download request missing id: {request}")
|
||||
|
||||
relative_path_raw = request.get("local_relative_path") or request.get("relative_path")
|
||||
filename = request.get("original_filename") or request.get("filename") or f"download_request_{request_id}.bin"
|
||||
relative_path = safe_relative_path(relative_path_raw, filename)
|
||||
local_file = resolve_under_root(self.config.storage_root, relative_path)
|
||||
if not local_file.exists() or not local_file.is_file():
|
||||
raise FileNotFoundError(f"Requested file is not found locally: {local_file}")
|
||||
|
||||
file_size = local_file.stat().st_size
|
||||
digest = sha256_file(local_file)
|
||||
expected_hash = request.get("sha256_hash") or request.get("file_hash")
|
||||
if expected_hash and str(expected_hash).lower() != digest.lower():
|
||||
raise RuntimeError(f"SHA256 mismatch before upload. expected={expected_hash} actual={digest}")
|
||||
|
||||
self.logger.info("Uploading file for download request=%s path=%s", request_id, local_file)
|
||||
extra = {
|
||||
"sha256_hash": digest,
|
||||
"file_hash": digest,
|
||||
"file_size": file_size,
|
||||
"local_relative_path": str(relative_path).replace("\\", "/"),
|
||||
}
|
||||
self.client.upload_download_request_file(request_id, local_file, extra)
|
||||
self.db.record_download_request(str(request_id), str(relative_path).replace("\\", "/"), digest, file_size)
|
||||
self.logger.info("Download request fulfilled request=%s size=%s sha256=%s", request_id, file_size, digest)
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import websockets
|
||||
|
||||
from .sync import StorageAgent
|
||||
|
||||
|
||||
class StorageAgentTunnel:
|
||||
def __init__(self, agent: StorageAgent):
|
||||
self.agent = agent
|
||||
self.config = agent.config
|
||||
self.logger = agent.logger
|
||||
|
||||
async def run_forever(self) -> None:
|
||||
self.logger.info("Starting tunnel mode for node=%s", self.config.node_code)
|
||||
while True:
|
||||
try:
|
||||
await self._connect_once()
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.exception("Tunnel disconnected/error: %s", exc)
|
||||
await asyncio.sleep(max(5, self.config.tunnel_reconnect_seconds))
|
||||
|
||||
async def _connect_once(self) -> None:
|
||||
async with websockets.connect(
|
||||
self.config.tunnel_url,
|
||||
ping_interval=30,
|
||||
ping_timeout=30,
|
||||
close_timeout=10,
|
||||
max_size=1024 * 1024,
|
||||
) as websocket:
|
||||
await websocket.send(self._json_text({"type": "ready", "agent_time_utc": self._now()}))
|
||||
self.agent._maybe_heartbeat(force=True)
|
||||
async for raw in websocket:
|
||||
message = self._parse_json(raw)
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
msg_type = message.get("type")
|
||||
if msg_type == "connected":
|
||||
self.logger.info("Tunnel connected: %s", message.get("node_code"))
|
||||
continue
|
||||
if msg_type == "sync":
|
||||
jobs = message.get("jobs") or []
|
||||
requests_ = message.get("download_requests") or message.get("requests") or []
|
||||
await asyncio.to_thread(self.agent.process_storage_jobs_from_payload, jobs)
|
||||
await asyncio.to_thread(self.agent.process_download_requests_from_payload, requests_)
|
||||
await websocket.send(self._json_text({
|
||||
"type": "agent_status",
|
||||
"jobs_seen": len(jobs),
|
||||
"requests_seen": len(requests_),
|
||||
"agent_time_utc": self._now(),
|
||||
}))
|
||||
continue
|
||||
if msg_type == "error":
|
||||
self.logger.error("Tunnel server error: %s", message.get("error"))
|
||||
|
||||
def _now(self) -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _json_text(self, payload: dict[str, Any]) -> str:
|
||||
import json
|
||||
return json.dumps(payload, separators=(",", ":"))
|
||||
|
||||
def _parse_json(self, raw: str) -> Any:
|
||||
import json
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
from . import __version__
|
||||
|
||||
|
||||
class AgentUpdater:
|
||||
def __init__(self, config, client, logger, install_dir: Path):
|
||||
self.config = config
|
||||
self.client = client
|
||||
self.logger = logger
|
||||
self.install_dir = install_dir.resolve()
|
||||
self.last_check = 0.0
|
||||
self.busy = False
|
||||
|
||||
def maybe_update(self, force: bool = False) -> bool:
|
||||
if not self.config.auto_update or self.busy:
|
||||
return False
|
||||
now = time.time()
|
||||
if not force and now - self.last_check < self.config.update_check_interval_seconds:
|
||||
return False
|
||||
self.last_check = now
|
||||
try:
|
||||
manifest = self.client.update_manifest()
|
||||
latest = str(manifest.get("latest_version") or "").strip()
|
||||
if not latest or latest == __version__:
|
||||
return False
|
||||
expected = str(manifest.get("sha256") or "").lower().strip()
|
||||
package = self.client.download_update_package()
|
||||
actual = hashlib.sha256(package).hexdigest().lower()
|
||||
if len(expected) != 64 or actual != expected:
|
||||
raise RuntimeError("ERP Local Agent update package SHA256 verification failed.")
|
||||
self._stage_and_restart(latest, package)
|
||||
return True
|
||||
except Exception as exc:
|
||||
self.logger.exception("ERP Local Agent update check failed: %s", exc)
|
||||
return False
|
||||
|
||||
def _stage_and_restart(self, latest: str, package: bytes) -> None:
|
||||
self.busy = True
|
||||
updates = self.install_dir / "updates"
|
||||
updates.mkdir(parents=True, exist_ok=True)
|
||||
safe = "".join(ch for ch in latest if ch.isalnum() or ch in ".-_") or "update"
|
||||
package_path = updates / f"ERP_Local_Agent_{safe}.zip"
|
||||
package_path.write_bytes(package)
|
||||
staged = updates / f"staged_{safe}"
|
||||
shutil.rmtree(staged, ignore_errors=True)
|
||||
staged.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(package_path, "r") as archive:
|
||||
archive.extractall(staged)
|
||||
if not (staged / "erp_local_agent" / "__init__.py").exists():
|
||||
raise RuntimeError("ERP Local Agent update package is incomplete.")
|
||||
script_path = updates / f"apply_{safe}.ps1"
|
||||
script_path.write_text(self._powershell_update_script(staged), encoding="utf-8")
|
||||
flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) | getattr(subprocess, "DETACHED_PROCESS", 0)
|
||||
subprocess.Popen(
|
||||
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(script_path), "-ParentPid", str(os.getpid())],
|
||||
cwd=str(self.install_dir), creationflags=flags, close_fds=True,
|
||||
)
|
||||
self.logger.warning("ERP Local Agent update %s staged; restarting.", latest)
|
||||
os._exit(0)
|
||||
|
||||
def _powershell_update_script(self, staged: Path) -> str:
|
||||
install = str(self.install_dir).replace("'", "''")
|
||||
stage = str(staged.resolve()).replace("'", "''")
|
||||
lines = [
|
||||
"param([int]$ParentPid)",
|
||||
'$ErrorActionPreference = "Stop"',
|
||||
"$InstallDir = '" + install + "'",
|
||||
"$StagedDir = '" + stage + "'",
|
||||
"$TaskName = 'ERP Local Agent'",
|
||||
"$BackupDir = Join-Path $InstallDir ('updates\\backup_' + (Get-Date -Format 'yyyyMMdd_HHmmss'))",
|
||||
"try {",
|
||||
" if ($ParentPid -gt 0) { try { Wait-Process -Id $ParentPid -Timeout 120 -ErrorAction SilentlyContinue } catch {} }",
|
||||
" try { Stop-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue } catch {}",
|
||||
" New-Item -ItemType Directory -Force -Path $BackupDir | Out-Null",
|
||||
" if (Test-Path (Join-Path $InstallDir 'erp_local_agent')) { Copy-Item (Join-Path $InstallDir 'erp_local_agent') -Destination $BackupDir -Recurse -Force }",
|
||||
" Remove-Item (Join-Path $InstallDir 'erp_local_agent') -Recurse -Force -ErrorAction SilentlyContinue",
|
||||
" Copy-Item (Join-Path $StagedDir 'erp_local_agent') -Destination $InstallDir -Recurse -Force",
|
||||
" foreach ($Item in @('requirements.txt','run_agent.bat','run_once.bat','check_config.py','check_config.bat','start_task_scheduler.bat','stop_task_scheduler.bat','status_task_scheduler.bat','install_task_scheduler.bat','uninstall_task_scheduler.bat')) { $Source=Join-Path $StagedDir $Item; if (Test-Path $Source) { Copy-Item $Source -Destination (Join-Path $InstallDir $Item) -Force } }",
|
||||
" $Python = Join-Path $InstallDir '.venv\\Scripts\\python.exe'",
|
||||
" if (-not (Test-Path $Python)) { throw 'ERP Local Agent virtual environment is missing.' }",
|
||||
" & $Python -m pip install -r (Join-Path $InstallDir 'requirements.txt')",
|
||||
" if ($LASTEXITCODE -ne 0) { throw 'Dependency update failed.' }",
|
||||
" Start-ScheduledTask -TaskName $TaskName",
|
||||
"} catch {",
|
||||
" try { if (Test-Path (Join-Path $BackupDir 'erp_local_agent')) { Remove-Item (Join-Path $InstallDir 'erp_local_agent') -Recurse -Force -ErrorAction SilentlyContinue; Copy-Item (Join-Path $BackupDir 'erp_local_agent') -Destination $InstallDir -Recurse -Force }; Start-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue } catch {}",
|
||||
" New-Item -ItemType Directory -Force -Path (Join-Path $InstallDir 'logs') | Out-Null",
|
||||
" Add-Content -Path (Join-Path $InstallDir 'logs\\update_error.log') -Value ((Get-Date).ToString('s') + ' ' + $_.Exception.Message)",
|
||||
" exit 1",
|
||||
"}",
|
||||
]
|
||||
return "\r\n".join(lines) + "\r\n"
|
||||
@@ -0,0 +1,21 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0"
|
||||
if not exist ".env" ( echo ERROR: .env file is missing. Download the configured ERP Local Agent package from ERP. & pause & exit /b 1 )
|
||||
if not exist ".venv\Scripts\python.exe" ( py -3 -m venv .venv & if errorlevel 1 ( echo ERROR: Unable to create Python virtual environment. & pause & exit /b 1 ) )
|
||||
".venv\Scripts\python.exe" -m pip install --upgrade pip
|
||||
if errorlevel 1 goto :failed
|
||||
".venv\Scripts\python.exe" -m pip install -r requirements.txt
|
||||
if errorlevel 1 goto :failed
|
||||
set "AGENT_DIR=%~dp0"
|
||||
set "AGENT_PYTHON=%~dp0.venv\Scripts\python.exe"
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "$ErrorActionPreference='Stop'; try { Stop-ScheduledTask -TaskName 'AuditFirmStorageAgent' -ErrorAction SilentlyContinue } catch {}; try { Unregister-ScheduledTask -TaskName 'AuditFirmStorageAgent' -Confirm:$false -ErrorAction SilentlyContinue } catch {}; $d=$env:AGENT_DIR.TrimEnd('\'); $p=$env:AGENT_PYTHON; $a=New-ScheduledTaskAction -Execute $p -Argument '-m erp_local_agent.main' -WorkingDirectory $d; $t=New-ScheduledTaskTrigger -AtStartup; $pr=New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest; $s=New-ScheduledTaskSettingsSet -RestartCount 999 -RestartInterval (New-TimeSpan -Minutes 1) -ExecutionTimeLimit ([TimeSpan]::Zero) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries; Register-ScheduledTask -TaskName 'ERP Local Agent' -Action $a -Trigger $t -Principal $pr -Settings $s -Force | Out-Null; Start-ScheduledTask -TaskName 'ERP Local Agent'"
|
||||
if errorlevel 1 goto :failed
|
||||
echo ERP Local Agent installed and started successfully.
|
||||
echo Future compatible releases will update automatically from ERP.
|
||||
pause
|
||||
exit /b 0
|
||||
:failed
|
||||
echo ERROR: ERP Local Agent installation failed. Run this file as Administrator.
|
||||
pause
|
||||
exit /b 1
|
||||
@@ -0,0 +1,3 @@
|
||||
requests==2.32.3
|
||||
python-dotenv==1.0.1
|
||||
websockets==12.0
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0"
|
||||
".venv\Scripts\python.exe" -m erp_local_agent.main
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0"
|
||||
".venv\Scripts\python.exe" -m erp_local_agent.main --once
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "Start-ScheduledTask -TaskName 'ERP Local Agent'"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "Get-ScheduledTask -TaskName 'ERP Local Agent' | Get-ScheduledTaskInfo | Format-List *"
|
||||
pause
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "Stop-ScheduledTask -TaskName 'ERP Local Agent'"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "Unregister-ScheduledTask -TaskName 'ERP Local Agent' -Confirm:$false -ErrorAction SilentlyContinue; Unregister-ScheduledTask -TaskName 'AuditFirmStorageAgent' -Confirm:$false -ErrorAction SilentlyContinue"
|
||||
pause
|
||||
Reference in New Issue
Block a user