diff --git a/alembic/versions/20260919_system_default_task_inheritance.py b/alembic/versions/20260919_system_default_task_inheritance.py new file mode 100644 index 0000000..0db2e38 --- /dev/null +++ b/alembic/versions/20260919_system_default_task_inheritance.py @@ -0,0 +1,80 @@ +"""System default task inheritance and firm update review. + +Revision ID: 20260919_task_inheritance +Revises: 20260918_accounting_mirror_ver +""" +from alembic import op +import sqlalchemy as sa + +revision = "20260919_task_inheritance" +down_revision = "20260918_accounting_mirror_ver" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + columns = {c["name"] for c in inspector.get_columns("firm_service_task_templates")} + fks = {fk.get("name") for fk in inspector.get_foreign_keys("firm_service_task_templates")} + + # batch_alter_table keeps this migration compatible with both production + # PostgreSQL and the project's supported SQLite development database. + with op.batch_alter_table("firm_service_task_templates") as batch: + if "source_system_task_id" not in columns: + batch.add_column(sa.Column("source_system_task_id", sa.Integer(), nullable=True)) + if "is_customized" not in columns: + batch.add_column(sa.Column("is_customized", sa.Boolean(), nullable=False, server_default=sa.false())) + if "last_synced_system_hash" not in columns: + batch.add_column(sa.Column("last_synced_system_hash", sa.String(length=64), nullable=True)) + if "last_reviewed_system_hash" not in columns: + batch.add_column(sa.Column("last_reviewed_system_hash", sa.String(length=64), nullable=True)) + if "system_update_available" not in columns: + batch.add_column(sa.Column("system_update_available", sa.Boolean(), nullable=False, server_default=sa.false())) + if "system_update_detected_at_utc" not in columns: + batch.add_column(sa.Column("system_update_detected_at_utc", sa.DateTime(timezone=True), nullable=True)) + if "fk_firm_task_source_system_task" not in fks: + batch.create_foreign_key( + "fk_firm_task_source_system_task", + "service_default_task_templates", + ["source_system_task_id"], + ["id"], + ondelete="SET NULL", + ) + + inspector = sa.inspect(bind) + indexes = {i.get("name") for i in inspector.get_indexes("firm_service_task_templates")} + if "ix_firm_service_task_templates_source_system_task_id" not in indexes: + op.create_index("ix_firm_service_task_templates_source_system_task_id", "firm_service_task_templates", ["source_system_task_id"], unique=False) + if "ix_firm_service_task_templates_is_customized" not in indexes: + op.create_index("ix_firm_service_task_templates_is_customized", "firm_service_task_templates", ["is_customized"], unique=False) + if "ix_firm_service_task_templates_system_update_available" not in indexes: + op.create_index("ix_firm_service_task_templates_system_update_available", "firm_service_task_templates", ["system_update_available"], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + indexes = {i.get("name") for i in inspector.get_indexes("firm_service_task_templates")} + if "ix_firm_service_task_templates_system_update_available" in indexes: + op.drop_index("ix_firm_service_task_templates_system_update_available", table_name="firm_service_task_templates") + if "ix_firm_service_task_templates_is_customized" in indexes: + op.drop_index("ix_firm_service_task_templates_is_customized", table_name="firm_service_task_templates") + if "ix_firm_service_task_templates_source_system_task_id" in indexes: + op.drop_index("ix_firm_service_task_templates_source_system_task_id", table_name="firm_service_task_templates") + + columns = {c["name"] for c in sa.inspect(bind).get_columns("firm_service_task_templates")} + fks = {fk.get("name") for fk in sa.inspect(bind).get_foreign_keys("firm_service_task_templates")} + with op.batch_alter_table("firm_service_task_templates") as batch: + if "fk_firm_task_source_system_task" in fks: + batch.drop_constraint("fk_firm_task_source_system_task", type_="foreignkey") + for name in ( + "system_update_detected_at_utc", + "system_update_available", + "last_reviewed_system_hash", + "last_synced_system_hash", + "is_customized", + "source_system_task_id", + ): + if name in columns: + batch.drop_column(name) diff --git a/app/modules/services/bulk_imports.py b/app/modules/services/bulk_imports.py index b59ebd7..25fe689 100644 --- a/app/modules/services/bulk_imports.py +++ b/app/modules/services/bulk_imports.py @@ -1066,6 +1066,7 @@ def import_system_default_tasks(db: Session, *, current_user, file_bytes: bytes, return {"created": 0, "updated": 0, "skipped": 0, "errors": [{"row": 1, "message": f"Missing columns: {', '.join(missing)}"}]} created = updated = skipped = 0 errors: list[dict] = [] + touched_catalogue_ids: set[int] = set() for row_no, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2): if not any(v not in (None, "") for v in row): continue @@ -1078,6 +1079,7 @@ def import_system_default_tasks(db: Session, *, current_user, file_bytes: bytes, catalogue = find_service(db, service_code=service_code) if not catalogue: raise ValueError("Service code not found in service catalogue.") + touched_catalogue_ids.add(int(catalogue.id)) if not sequence_no or not task_name: raise ValueError("sequence_no and task_name are required.") task = db.execute( @@ -1128,11 +1130,42 @@ def import_system_default_tasks(db: Session, *, current_user, file_bytes: bytes, task.is_active = _bool(_cell(row, headers, "is_active"), True) except Exception as exc: errors.append({"row": row_no, "message": str(exc)}) + rollout_summary = { + "firms_processed": 0, + "firms_changed": 0, + "custom_updates_available": 0, + "engagement_created": 0, + "engagement_updated_pending": 0, + "engagement_deactivated_pending": 0, + } if not errors: + # Apply the same transaction to every enabled firm and safely refresh only + # unstarted tasks in open/unlocked engagements. Customized firm tasks are + # never overwritten; they receive an update-available decision instead. + from app.modules.services.default_task_sync import sync_system_defaults_to_all_firms + for catalogue_id in sorted(touched_catalogue_ids): + rollout = sync_system_defaults_to_all_firms( + db, + service_catalogue_id=catalogue_id, + updated_by_user_id=current_user.id, + sync_open_engagements=True, + ) + rollout_summary["firms_processed"] += rollout.firms_processed + rollout_summary["firms_changed"] += rollout.firms_changed + rollout_summary["custom_updates_available"] += rollout.custom_updates_available + rollout_summary["engagement_created"] += rollout.engagement_created + rollout_summary["engagement_updated_pending"] += rollout.engagement_updated_pending + rollout_summary["engagement_deactivated_pending"] += rollout.engagement_deactivated_pending db.commit() else: db.rollback() - return {"created": created if not errors else 0, "updated": updated if not errors else 0, "skipped": skipped, "errors": errors} + return { + "created": created if not errors else 0, + "updated": updated if not errors else 0, + "skipped": skipped, + "errors": errors, + **(rollout_summary if not errors else {}), + } def import_firm_task_templates(db: Session, *, current_user, tenant_id: int, file_bytes: bytes, update_existing: bool = True, expected_service_code: str | None = None) -> dict: @@ -1210,6 +1243,9 @@ def import_firm_task_templates(db: Session, *, current_user, tenant_id: int, fil task.aqmm_blocks_final_release = _bool(_cell(row, headers, "aqmm_blocks_final_release"), False) task.aqmm_reference = _clean(_cell(row, headers, "aqmm_reference")) or None task.is_active = _bool(_cell(row, headers, "is_active"), True) + # A firm-level workbook import is an explicit firm customization. Linked + # system defaults remain available for later Firm Admin upgrade review. + task.is_customized = True task.updated_by_user_id = current_user.id except Exception as exc: errors.append({"row": row_no, "message": str(exc)}) diff --git a/app/modules/services/default_task_sync.py b/app/modules/services/default_task_sync.py index 967571d..1975862 100644 --- a/app/modules/services/default_task_sync.py +++ b/app/modules/services/default_task_sync.py @@ -1,129 +1,285 @@ from __future__ import annotations -from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field +from datetime import datetime, timezone +import hashlib +import json import re from typing import Any from sqlalchemy import func, select -from sqlalchemy.inspection import inspect as sa_inspect from sqlalchemy.orm import Session from app.modules.services.models import ( ClientServiceTaskInstance, + FirmServiceSelection, FirmServiceTaskTemplate, ServiceDefaultTaskTemplate, + ServiceTaskCategory, ) +# Fields that define the centrally maintained system-default task snapshot. +# task_category_id is deliberately excluded because system and firm category +# masters use different scoped rows; task_category text is the portable value. +SYSTEM_TASK_FIELDS: tuple[str, ...] = ( + "task_name", + "description", + "sequence_no", + "default_role_name", + "eligible_role_names", + "is_mandatory", + "requires_review", + "normal_review_role", + "task_category", + "response_required", + "response_type", + "evidence_required", + "remarks_required_if_no", + "task_tool_code", + "is_aqmm_task", + "aqmm_mandatory", + "aqmm_evidence_required", + "aqmm_manager_review_required", + "aqmm_partner_review_required", + "aqmm_review_partner_required", + "aqmm_blocks_final_release", + "aqmm_reference", + "is_active", +) + +_SPACE_RE = re.compile(r"\s+") + + @dataclass class FirmDefaultTaskSyncResult: created: int = 0 updated: int = 0 duplicates_disabled: int = 0 unchanged: int = 0 + custom_updates_available: int = 0 + linked_legacy: int = 0 + engagement_created: int = 0 + engagement_updated_pending: int = 0 + engagement_deactivated_pending: int = 0 + engagement_preserved_history: int = 0 @property def active_total_change(self) -> int: return self.created - self.duplicates_disabled -_SPACE_RE = re.compile(r"\s+") +@dataclass +class SystemDefaultRolloutResult: + firms_processed: int = 0 + firms_changed: int = 0 + created: int = 0 + updated: int = 0 + unchanged: int = 0 + custom_updates_available: int = 0 + duplicates_disabled: int = 0 + engagement_created: int = 0 + engagement_updated_pending: int = 0 + engagement_deactivated_pending: int = 0 + engagement_preserved_history: int = 0 + tenant_results: dict[int, FirmDefaultTaskSyncResult] = field(default_factory=dict) def _normalise_name(value: str | None) -> str: return _SPACE_RE.sub(" ", (value or "").strip()).casefold() -def _column_names(model: type[Any]) -> set[str]: - return {column.key for column in sa_inspect(model).mapper.column_attrs} +def _portable_value(value: Any) -> Any: + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def _snapshot_dict(row: Any) -> dict[str, Any]: + return {name: _portable_value(getattr(row, name, None)) for name in SYSTEM_TASK_FIELDS} + + +def system_task_hash(source: ServiceDefaultTaskTemplate) -> str: + payload = json.dumps(_snapshot_dict(source), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def firm_task_hash(target: FirmServiceTaskTemplate) -> str: + payload = json.dumps(_snapshot_dict(target), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def system_task_diff(source: ServiceDefaultTaskTemplate, target: FirmServiceTaskTemplate) -> list[dict[str, Any]]: + labels = { + "task_name": "Task name", + "description": "Description", + "sequence_no": "Sequence", + "default_role_name": "Default role", + "eligible_role_names": "Eligible roles", + "is_mandatory": "Mandatory", + "requires_review": "Review required", + "normal_review_role": "Normal reviewer", + "task_category": "Task category", + "response_required": "Response required", + "response_type": "Response type", + "evidence_required": "Evidence required", + "remarks_required_if_no": "Remarks required if No", + "task_tool_code": "Task tool", + "is_aqmm_task": "AQMM task", + "aqmm_mandatory": "AQMM mandatory", + "aqmm_evidence_required": "AQMM evidence", + "aqmm_manager_review_required": "AQMM manager review", + "aqmm_partner_review_required": "AQMM partner review", + "aqmm_review_partner_required": "AQMM review partner", + "aqmm_blocks_final_release": "Blocks final release", + "aqmm_reference": "AQMM reference", + "is_active": "Active", + } + changes: list[dict[str, Any]] = [] + for name in SYSTEM_TASK_FIELDS: + old = getattr(target, name, None) + new = getattr(source, name, None) + if old != new: + changes.append({"field": name, "label": labels.get(name, name), "firm": old, "system": new}) + return changes + + +def mark_firm_task_customized(task: FirmServiceTaskTemplate) -> None: + """Mark an explicitly edited firm task as protected from automatic overwrite.""" + task.is_customized = True + # Do not clear an already pending system update. If there is no pending update, + # the next system revision/hash change will create one automatically. + + +def _ensure_firm_category( + db: Session, + *, + source: ServiceDefaultTaskTemplate, + tenant_id: int, + user_id: int | None, +) -> ServiceTaskCategory | None: + name = (getattr(source, "task_category", None) or "").strip() + if not name: + return None + + system_category = None + source_category_id = getattr(source, "task_category_id", None) + if source_category_id: + system_category = db.get(ServiceTaskCategory, int(source_category_id)) + + code = ((getattr(system_category, "code", None) or "").strip().upper() if system_category else "") + if code: + existing = db.execute( + select(ServiceTaskCategory).where( + ServiceTaskCategory.tenant_id == tenant_id, + ServiceTaskCategory.service_catalogue_id == source.service_catalogue_id, + ServiceTaskCategory.code == code, + ) + ).scalar_one_or_none() + else: + existing = db.execute( + select(ServiceTaskCategory).where( + ServiceTaskCategory.tenant_id == tenant_id, + ServiceTaskCategory.service_catalogue_id == source.service_catalogue_id, + func.lower(ServiceTaskCategory.name) == name.lower(), + ) + ).scalar_one_or_none() + + if existing: + # Central category rename/status/order should remain aligned for inherited use. + # Firm category rows are shared by firm templates, so update only metadata that + # does not destroy task history. + if system_category: + existing.name = system_category.name + existing.sort_order = system_category.sort_order + existing.is_active = system_category.is_active + if user_id is not None: + existing.updated_by_user_id = user_id + return existing + + if not code: + # Match the application's existing category-code convention sufficiently for + # automatic inherited categories without importing services.py (avoids cycles). + code = re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")[:50] or "CATEGORY" + base = code + suffix = 2 + while db.execute( + select(ServiceTaskCategory.id).where( + ServiceTaskCategory.tenant_id == tenant_id, + ServiceTaskCategory.service_catalogue_id == source.service_catalogue_id, + ServiceTaskCategory.code == code, + ) + ).first(): + code = f"{base[:45]}_{suffix}" + suffix += 1 + + row = ServiceTaskCategory( + tenant_id=tenant_id, + service_catalogue_id=source.service_catalogue_id, + code=code, + name=(system_category.name if system_category else name), + sort_order=(system_category.sort_order if system_category else 100), + is_active=(system_category.is_active if system_category else True), + created_by_user_id=user_id, + updated_by_user_id=user_id, + ) + db.add(row) + db.flush() + return row def _copy_default_columns( + db: Session, source: ServiceDefaultTaskTemplate, target: FirmServiceTaskTemplate, + *, + tenant_id: int, + user_id: int | None, + allow_sequence_change: bool = True, ) -> bool: - """Copy only columns that exist on both system and firm task models. - - Firm-only identity/audit fields are deliberately excluded. Related firm - requirement/template-file records are untouched. - """ - source_cols = _column_names(ServiceDefaultTaskTemplate) - target_cols = _column_names(FirmServiceTaskTemplate) - - excluded = { - "id", - "tenant_id", - "service_catalogue_id", - "created_at", - "created_at_utc", - "created_by_user_id", - "updated_at", - "updated_at_utc", - "updated_by_user_id", - } - changed = False - for name in sorted((source_cols & target_cols) - excluded): + for name in SYSTEM_TASK_FIELDS: + if name == "sequence_no" and not allow_sequence_change: + continue new_value = getattr(source, name, None) if getattr(target, name, None) != new_value: setattr(target, name, new_value) changed = True - # IMPORTANT: - # is_active is copied from ServiceDefaultTaskTemplate above because it is - # a shared mapped column. Do NOT force FirmServiceTaskTemplate back to True. - # - # This makes System Default activation state authoritative during sync: - # System Default disabled -> Firm Task disabled - # System Default enabled -> Firm Task enabled - # - # Firm-only/custom tasks remain untouched because this function is called - # only for tasks matched to current System Defaults. + category = _ensure_firm_category(db, source=source, tenant_id=tenant_id, user_id=user_id) + category_id = category.id if category else None + category_name = category.name if category else None + if getattr(target, "task_category_id", None) != category_id: + target.task_category_id = category_id + changed = True + if getattr(target, "task_category", None) != category_name: + target.task_category = category_name + changed = True return changed -def _reference_counts( - db: Session, - *, - firm_task_ids: list[int], -) -> dict[int, int]: +def _reference_counts(db: Session, *, firm_task_ids: list[int]) -> dict[int, int]: if not firm_task_ids: return {} - rows = db.execute( - select( - ClientServiceTaskInstance.firm_task_template_id, - func.count(ClientServiceTaskInstance.id), - ) + select(ClientServiceTaskInstance.firm_task_template_id, func.count(ClientServiceTaskInstance.id)) .where(ClientServiceTaskInstance.firm_task_template_id.in_(firm_task_ids)) .group_by(ClientServiceTaskInstance.firm_task_template_id) ).all() - - return { - int(template_id): int(count) - for template_id, count in rows - if template_id is not None - } + return {int(template_id): int(count) for template_id, count in rows if template_id is not None} -def _choose_canonical( - candidates: list[FirmServiceTaskTemplate], - *, - reference_counts: dict[int, int], -) -> FirmServiceTaskTemplate: - """Prefer the row already used by the most engagement task instances. +def _choose_canonical(candidates: list[FirmServiceTaskTemplate], *, reference_counts: dict[int, int]) -> FirmServiceTaskTemplate: + return sorted(candidates, key=lambda row: (-reference_counts.get(int(row.id), 0), int(row.id)))[0] - This avoids breaking historical links. If usage is equal, keep the oldest - row (smallest id). - """ - return sorted( - candidates, - key=lambda row: ( - -reference_counts.get(int(row.id), 0), - int(row.id), - ), - )[0] + +def _next_free_sequence(used: set[int], preferred: int) -> int: + if preferred > 0 and preferred not in used: + return preferred + candidate = max(used or {0}) + 1 + while candidate in used: + candidate += 1 + return candidate def sync_firm_tasks_from_system_defaults( @@ -132,183 +288,305 @@ def sync_firm_tasks_from_system_defaults( tenant_id: int, service_catalogue_id: int, updated_by_user_id: int | None = None, + sync_open_engagements: bool = False, ) -> FirmDefaultTaskSyncResult: - """Idempotently synchronise system default tasks into one firm's template set. + """Synchronise one firm's task templates from centrally maintained defaults. - Behaviour: - * existing firm task with the same normalised task name -> UPDATE, never INSERT; - * duplicate firm rows with the same task name -> retain one canonical active row, - mark the additional template rows inactive; - * missing system default -> create one firm task; - * unique firm-only/custom tasks are preserved; - * existing ClientServiceTaskInstance rows are never deleted or reassigned. - - The function may safely be called repeatedly. + Inherited tasks are updated automatically. Explicitly customized linked tasks are + never overwritten; a system_update_available flag is raised for Firm Admin review. + Firm-only tasks remain untouched. Legacy rows are linked by exact normalized name. """ defaults = db.execute( select(ServiceDefaultTaskTemplate) - .where( - ServiceDefaultTaskTemplate.service_catalogue_id == service_catalogue_id, - ) - .order_by( - ServiceDefaultTaskTemplate.sequence_no.asc(), - ServiceDefaultTaskTemplate.id.asc(), - ) + .where(ServiceDefaultTaskTemplate.service_catalogue_id == service_catalogue_id) + .order_by(ServiceDefaultTaskTemplate.sequence_no.asc(), ServiceDefaultTaskTemplate.id.asc()) ).scalars().all() - firm_rows = db.execute( select(FirmServiceTaskTemplate) .where( FirmServiceTaskTemplate.tenant_id == tenant_id, FirmServiceTaskTemplate.service_catalogue_id == service_catalogue_id, ) - .order_by( - FirmServiceTaskTemplate.sequence_no.asc(), - FirmServiceTaskTemplate.id.asc(), - ) + .order_by(FirmServiceTaskTemplate.sequence_no.asc(), FirmServiceTaskTemplate.id.asc()) ).scalars().all() result = FirmDefaultTaskSyncResult() - - by_name: dict[str, list[FirmServiceTaskTemplate]] = defaultdict(list) + reference_counts = _reference_counts(db, firm_task_ids=[int(r.id) for r in firm_rows if r.id]) + by_source: dict[int, list[FirmServiceTaskTemplate]] = {} + legacy_by_name: dict[str, list[FirmServiceTaskTemplate]] = {} for row in firm_rows: - key = _normalise_name(getattr(row, "task_name", None)) - if key: - by_name[key].append(row) + if row.source_system_task_id: + by_source.setdefault(int(row.source_system_task_id), []).append(row) + else: + key = _normalise_name(row.task_name) + if key: + legacy_by_name.setdefault(key, []).append(row) - reference_counts = _reference_counts( - db, - firm_task_ids=[int(row.id) for row in firm_rows if getattr(row, "id", None)], - ) + used_sequences = {int(r.sequence_no) for r in firm_rows if r.sequence_no is not None} - # ------------------------------------------------------------------ - # IMPORTANT: two-phase sequence re-numbering - # - # The database enforces a UNIQUE constraint on: - # (tenant_id, service_catalogue_id, sequence_no) - # - # Existing duplicate imports may contain several generations of the same - # task at different sequence numbers. During a sync, moving a canonical - # row directly onto the latest system-default sequence can therefore - # collide with another historical row that still owns that sequence. - # - # Marking that historical row inactive does NOT release the unique key. - # Park every existing firm template on a guaranteed-unique temporary - # sequence first, flush, and only then apply final/default sequences. - # ------------------------------------------------------------------ - max_existing_sequence = max( - [int(getattr(row, "sequence_no", 0) or 0) for row in firm_rows] + [0] - ) - max_default_sequence = max( - [int(getattr(row, "sequence_no", 0) or 0) for row in defaults] + [0] - ) - parking_base = max(max_existing_sequence, max_default_sequence, 0) + 100000 - - for offset, row in enumerate(sorted(firm_rows, key=lambda item: int(item.id)), start=1): - row.sequence_no = parking_base + offset - - # This flush is intentional and must happen before any canonical row is - # assigned a system-default sequence number. - db.flush() - - # First clean pre-existing duplicate groups, regardless of whether the task - # remains in the current system defaults. A unique firm-only task is untouched. - for candidates in by_name.values(): - if len(candidates) <= 1: - continue - canonical = _choose_canonical( - candidates, - reference_counts=reference_counts, - ) - for duplicate in candidates: - if duplicate.id == canonical.id: - continue - - # Duplicate rows are retained for historical FK safety but remain - # on their unique parked sequence number. The UNIQUE sequence - # constraint therefore remains satisfied even after deactivation. - if getattr(duplicate, "is_active", True): - duplicate.is_active = False - if ( - updated_by_user_id is not None - and hasattr(duplicate, "updated_by_user_id") - ): - duplicate.updated_by_user_id = updated_by_user_id - result.duplicates_disabled += 1 - - # Upsert current system defaults. - for default in defaults: - key = _normalise_name(getattr(default, "task_name", None)) - candidates = by_name.get(key, []) if key else [] + for source in defaults: + latest_hash = system_task_hash(source) + candidates = by_source.get(int(source.id), []) + linked_from_legacy = False + if not candidates: + legacy_candidates = legacy_by_name.get(_normalise_name(source.task_name), []) + if not legacy_candidates: + # One-time migration fallback for a firm that renamed a previously + # copied default before provenance columns existed. Sequence is used + # only when there is exactly one unlinked candidate at that position. + legacy_candidates = [ + row for row in firm_rows + if row.source_system_task_id is None + and int(row.sequence_no or 0) == int(source.sequence_no or 0) + ] + if len(legacy_candidates) != 1: + legacy_candidates = [] + if legacy_candidates: + target = _choose_canonical(legacy_candidates, reference_counts=reference_counts) + candidates = [target] + target.source_system_task_id = source.id + linked_from_legacy = True + result.linked_legacy += 1 + by_source.setdefault(int(source.id), []).append(target) + # Remove from future legacy matching. + key = _normalise_name(target.task_name) + if key in legacy_by_name: + legacy_by_name[key] = [r for r in legacy_by_name[key] if r.id != target.id] if candidates: - target = _choose_canonical( - candidates, - reference_counts=reference_counts, - ) - changed = _copy_default_columns(default, target) + target = _choose_canonical(candidates, reference_counts=reference_counts) + for duplicate in candidates: + if duplicate.id == target.id: + continue + if duplicate.is_active: + duplicate.is_active = False + if updated_by_user_id is not None: + duplicate.updated_by_user_id = updated_by_user_id + result.duplicates_disabled += 1 + + # For legacy rows with no inheritance history, infer whether they were + # already firm-customized by comparing the full portable snapshot. + if linked_from_legacy and target.last_synced_system_hash is None: + target.is_customized = firm_task_hash(target) != latest_hash + if not target.is_customized: + target.last_synced_system_hash = latest_hash + target.last_reviewed_system_hash = latest_hash + + # Detect out-of-band edits to a linked inherited row as customization. if ( - updated_by_user_id is not None - and hasattr(target, "updated_by_user_id") + not target.is_customized + and target.last_synced_system_hash + and firm_task_hash(target) != target.last_synced_system_hash ): + target.is_customized = True + + if target.is_customized: + pending = target.last_reviewed_system_hash != latest_hash + target.system_update_available = pending + target.system_update_detected_at_utc = datetime.now(timezone.utc) if pending else None + if pending: + result.custom_updates_available += 1 + else: + result.unchanged += 1 + continue + + # Inherited task: copy system values. Sequence changes are applied when + # the desired number is free or currently owned by this same row. If a + # firm-only/custom row owns it, preserve that row and place this inherited + # task at the next free sequence instead of overwriting customization. + desired_seq = int(source.sequence_no or 0) + current_seq = int(target.sequence_no or 0) + allow_seq = desired_seq == current_seq or desired_seq not in (used_sequences - {current_seq}) + if allow_seq: + used_sequences.discard(current_seq) + used_sequences.add(desired_seq) + changed = _copy_default_columns( + db, + source, + target, + tenant_id=tenant_id, + user_id=updated_by_user_id, + allow_sequence_change=allow_seq, + ) + target.source_system_task_id = source.id + target.last_synced_system_hash = latest_hash + target.last_reviewed_system_hash = latest_hash + target.system_update_available = False + target.system_update_detected_at_utc = None + if updated_by_user_id is not None: target.updated_by_user_id = updated_by_user_id - if changed: - result.updated += 1 - else: - result.unchanged += 1 + result.updated += 1 if changed else 0 + result.unchanged += 0 if changed else 1 continue - # No name match. A unique sequence match is a conservative fallback for a - # system-default rename while still avoiding arbitrary replacement. - # Do NOT use the pre-sync sequence number as an identity fallback here. - # All existing rows have deliberately been parked on temporary sequence - # numbers to satisfy the DB unique constraint. More importantly, - # sequence number is ordering metadata, not a stable task identity. - # A renamed system-default task without a name match is therefore - # treated as a new task rather than risking replacement of the wrong - # firm-specific task. - - # Missing task: construct from the intersection of mapped columns. - source_cols = _column_names(ServiceDefaultTaskTemplate) - target_cols = _column_names(FirmServiceTaskTemplate) - excluded = { - "id", - "tenant_id", - "service_catalogue_id", - "created_at", - "created_at_utc", - "created_by_user_id", - "updated_at", - "updated_at_utc", - "updated_by_user_id", - } - payload = { - name: getattr(default, name, None) - for name in sorted((source_cols & target_cols) - excluded) - } - payload["tenant_id"] = tenant_id - payload["service_catalogue_id"] = service_catalogue_id - if "is_active" in target_cols: - payload["is_active"] = True - if ( - updated_by_user_id is not None - and "created_by_user_id" in target_cols - ): - payload["created_by_user_id"] = updated_by_user_id - if ( - updated_by_user_id is not None - and "updated_by_user_id" in target_cols - ): - payload["updated_by_user_id"] = updated_by_user_id - - new_row = FirmServiceTaskTemplate(**payload) - db.add(new_row) + # Missing default: create a new inherited row. Avoid colliding with a + # firm-only sequence; identity is source_system_task_id, not sequence number. + preferred = int(source.sequence_no or 0) + seq = _next_free_sequence(used_sequences, preferred) + used_sequences.add(seq) + row = FirmServiceTaskTemplate( + tenant_id=tenant_id, + service_catalogue_id=service_catalogue_id, + task_name=source.task_name, + sequence_no=seq, + source_system_task_id=source.id, + is_customized=False, + last_synced_system_hash=latest_hash, + last_reviewed_system_hash=latest_hash, + system_update_available=False, + created_by_user_id=updated_by_user_id, + updated_by_user_id=updated_by_user_id, + ) + db.add(row) db.flush() - - firm_rows.append(new_row) - if key: - by_name[key].append(new_row) + _copy_default_columns( + db, + source, + row, + tenant_id=tenant_id, + user_id=updated_by_user_id, + allow_sequence_change=(seq == preferred), + ) result.created += 1 + firm_rows.append(row) + by_source.setdefault(int(source.id), []).append(row) db.flush() + + if sync_open_engagements: + from app.modules.services.execution import sync_open_engagement_tasks_for_service + + engagement = sync_open_engagement_tasks_for_service( + db, + tenant_id=tenant_id, + catalogue_id=service_catalogue_id, + user_id=updated_by_user_id or 0, + include_started_open_tasks=False, + safe_system_rollout=True, + ) + result.engagement_created = engagement.get("created", 0) + result.engagement_updated_pending = engagement.get("updated_pending", 0) + result.engagement_deactivated_pending = engagement.get("deactivated_pending", 0) + result.engagement_preserved_history = engagement.get("preserved_history", 0) + return result + + +def sync_system_defaults_to_all_firms( + db: Session, + *, + service_catalogue_id: int, + updated_by_user_id: int | None = None, + sync_open_engagements: bool = True, +) -> SystemDefaultRolloutResult: + """Roll a system-default service checklist to every firm that enabled it.""" + selections = db.execute( + select(FirmServiceSelection).where( + FirmServiceSelection.service_catalogue_id == service_catalogue_id, + FirmServiceSelection.is_enabled.is_(True), + ) + ).scalars().all() + + aggregate = SystemDefaultRolloutResult() + for selection in selections: + result = sync_firm_tasks_from_system_defaults( + db, + tenant_id=int(selection.tenant_id), + service_catalogue_id=service_catalogue_id, + updated_by_user_id=updated_by_user_id, + sync_open_engagements=sync_open_engagements, + ) + aggregate.firms_processed += 1 + if result.created or result.updated or result.duplicates_disabled or result.custom_updates_available: + aggregate.firms_changed += 1 + aggregate.created += result.created + aggregate.updated += result.updated + aggregate.unchanged += result.unchanged + aggregate.custom_updates_available += result.custom_updates_available + aggregate.duplicates_disabled += result.duplicates_disabled + aggregate.engagement_created += result.engagement_created + aggregate.engagement_updated_pending += result.engagement_updated_pending + aggregate.engagement_deactivated_pending += result.engagement_deactivated_pending + aggregate.engagement_preserved_history += result.engagement_preserved_history + aggregate.tenant_results[int(selection.tenant_id)] = result + return aggregate + + +def accept_system_update_for_firm_task( + db: Session, + *, + task: FirmServiceTaskTemplate, + user_id: int, + sync_open_engagements: bool = True, +) -> FirmDefaultTaskSyncResult: + if not task.source_system_task_id: + raise ValueError("This firm task is not linked to a system default.") + source = db.get(ServiceDefaultTaskTemplate, int(task.source_system_task_id)) + if source is None: + raise ValueError("The linked system default no longer exists.") + + latest_hash = system_task_hash(source) + used = { + int(v) + for v in db.scalars( + select(FirmServiceTaskTemplate.sequence_no).where( + FirmServiceTaskTemplate.tenant_id == task.tenant_id, + FirmServiceTaskTemplate.service_catalogue_id == task.service_catalogue_id, + FirmServiceTaskTemplate.id != task.id, + ) + ).all() + if v is not None + } + desired = int(source.sequence_no or 0) + allow_seq = desired not in used + changed = _copy_default_columns( + db, + source, + task, + tenant_id=int(task.tenant_id), + user_id=user_id, + allow_sequence_change=allow_seq, + ) + task.is_customized = False + task.last_synced_system_hash = latest_hash + task.last_reviewed_system_hash = latest_hash + task.system_update_available = False + task.system_update_detected_at_utc = None + task.updated_by_user_id = user_id + db.flush() + + result = FirmDefaultTaskSyncResult(updated=1 if changed else 0, unchanged=0 if changed else 1) + if sync_open_engagements: + from app.modules.services.execution import sync_open_engagement_tasks_for_service + engagement = sync_open_engagement_tasks_for_service( + db, + tenant_id=int(task.tenant_id), + catalogue_id=int(task.service_catalogue_id), + user_id=user_id, + include_started_open_tasks=False, + safe_system_rollout=True, + ) + result.engagement_created = engagement.get("created", 0) + result.engagement_updated_pending = engagement.get("updated_pending", 0) + result.engagement_deactivated_pending = engagement.get("deactivated_pending", 0) + result.engagement_preserved_history = engagement.get("preserved_history", 0) + return result + + +def keep_firm_customization_for_system_revision( + db: Session, + *, + task: FirmServiceTaskTemplate, + user_id: int, +) -> None: + if not task.source_system_task_id: + raise ValueError("This firm task is not linked to a system default.") + source = db.get(ServiceDefaultTaskTemplate, int(task.source_system_task_id)) + if source is None: + raise ValueError("The linked system default no longer exists.") + task.is_customized = True + task.last_reviewed_system_hash = system_task_hash(source) + task.system_update_available = False + task.system_update_detected_at_utc = None + task.updated_by_user_id = user_id + db.flush() diff --git a/app/modules/services/execution.py b/app/modules/services/execution.py index df61032..b19d5fe 100644 --- a/app/modules/services/execution.py +++ b/app/modules/services/execution.py @@ -442,16 +442,18 @@ def sync_open_engagement_tasks_for_service( catalogue_id: int, user_id: int, include_started_open_tasks: bool = False, + safe_system_rollout: bool = False, ) -> dict[str, int]: - templates = db.execute( + all_templates = db.execute( select(FirmServiceTaskTemplate) .where( FirmServiceTaskTemplate.tenant_id == tenant_id, FirmServiceTaskTemplate.service_catalogue_id == catalogue_id, - FirmServiceTaskTemplate.is_active.is_(True), ) .order_by(FirmServiceTaskTemplate.sequence_no.asc(), FirmServiceTaskTemplate.id.asc()) ).scalars().all() + templates = [row for row in all_templates if bool(row.is_active)] + inactive_templates = [row for row in all_templates if not bool(row.is_active)] engagements = db.execute( select(ClientServiceSubscription) @@ -470,10 +472,21 @@ def sync_open_engagement_tasks_for_service( "created": 0, "updated_pending": 0, "updated_started": 0, + "deactivated_pending": 0, "preserved_history": 0, } for subscription in engagements: + if ( + safe_system_rollout + and quality_required_for_engagement(subscription.engagement_type) + and getattr(subscription, "quality_acceptance_status", None) != QUALITY_APPROVED + ): + # Preserve the existing assurance acceptance gate during automatic + # system-default rollout. Tasks will be generated by the normal flow + # once quality acceptance is approved. + continue + existing = db.execute( select(ClientServiceTaskInstance).where( ClientServiceTaskInstance.subscription_id == subscription.id, @@ -549,6 +562,7 @@ def sync_open_engagement_tasks_for_service( and task.completed_at_utc is None and task.submitted_for_review_at_utc is None and (task.rework_status or "none") == "none" + and not bool(getattr(task, "is_locked", False)) ) if never_started: @@ -558,6 +572,8 @@ def sync_open_engagement_tasks_for_service( subscription=subscription, update_assignment=True, ) + if safe_system_rollout: + task.is_active = True task.updated_by_user_id = user_id result["updated_pending"] += 1 elif include_started_open_tasks and status in {"in_progress", "blocked"} and task.completed_at_utc is None: @@ -571,12 +587,35 @@ def sync_open_engagement_tasks_for_service( task.updated_by_user_id = user_id result["updated_started"] += 1 else: - # Preserve task execution/history, but allow an explicit template sync to - # update only the separated normal-review configuration for existing tasks. - _sync_normal_review_snapshot_preserving_history(task, template=template) - task.updated_by_user_id = user_id + if not safe_system_rollout: + # Preserve the pre-existing explicit/manual sync behaviour. + _sync_normal_review_snapshot_preserving_history(task, template=template) + task.updated_by_user_id = user_id + # Automatic system rollout never rewrites a started/completed/reviewed + # task snapshot. Audit/review history remains exactly as performed. result["preserved_history"] += 1 + if safe_system_rollout and inactive_templates: + inactive_ids = {int(row.id) for row in inactive_templates} + for task in existing: + if task.firm_task_template_id is None or int(task.firm_task_template_id) not in inactive_ids: + continue + status = (task.status or "pending").strip().lower() + never_started = ( + status == "pending" + and task.started_at_utc is None + and task.completed_at_utc is None + and task.submitted_for_review_at_utc is None + and (task.rework_status or "none") == "none" + and not bool(getattr(task, "is_locked", False)) + ) + if never_started and bool(getattr(task, "is_active", True)): + task.is_active = False + task.updated_by_user_id = user_id + result["deactivated_pending"] += 1 + elif not never_started: + result["preserved_history"] += 1 + return result diff --git a/app/modules/services/models.py b/app/modules/services/models.py index 88a6a90..0a51cb7 100644 --- a/app/modules/services/models.py +++ b/app/modules/services/models.py @@ -242,6 +242,17 @@ class FirmServiceTaskTemplate(CommonBase): is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + # System-default inheritance metadata. Firm-only tasks keep source_system_task_id=None. + # Linked tasks remain centrally synchronized until the firm explicitly customizes them. + source_system_task_id: Mapped[int | None] = mapped_column( + ForeignKey("service_default_task_templates.id", ondelete="SET NULL"), nullable=True, index=True + ) + is_customized: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + last_synced_system_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + last_reviewed_system_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + system_update_available: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + system_update_detected_at_utc: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) updated_by_user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True) @@ -256,6 +267,7 @@ class FirmServiceTaskTemplate(CommonBase): ) catalogue = relationship("ServiceCatalogue", back_populates="task_templates") + source_system_task = relationship("ServiceDefaultTaskTemplate", foreign_keys=[source_system_task_id]) document_requirements = relationship( "FirmTaskDocumentRequirement", back_populates="task_template", diff --git a/app/modules/services/task_categories_excel.py b/app/modules/services/task_categories_excel.py index a48fec3..cfa0457 100644 --- a/app/modules/services/task_categories_excel.py +++ b/app/modules/services/task_categories_excel.py @@ -183,6 +183,7 @@ def import_task_categories( created = 0 updated = 0 errors: list[dict] = [] + touched_service_ids: set[int] = set() expected = normalize_code(expected_service_code) if expected_service_code else "" for row_no, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2): @@ -205,6 +206,7 @@ def import_task_categories( ).scalar_one_or_none() if service is None: raise ValueError("Service code not found in Service Catalogue.") + touched_service_ids.add(int(service.id)) scope_filter = ( ServiceTaskCategory.tenant_id.is_(None) @@ -275,11 +277,22 @@ def import_task_categories( if old_name != category_name: for task in linked: task.task_category = category_name + if tenant_id is not None: + task.is_customized = True except Exception as exc: errors.append({"row": row_no, "message": str(exc)}) if errors: db.rollback() return {"created": 0, "updated": 0, "errors": errors} + if tenant_id is None and touched_service_ids: + from app.modules.services.default_task_sync import sync_system_defaults_to_all_firms + for service_id in sorted(touched_service_ids): + sync_system_defaults_to_all_firms( + db, + service_catalogue_id=service_id, + updated_by_user_id=current_user.id, + sync_open_engagements=True, + ) db.commit() return {"created": created, "updated": updated, "errors": []} diff --git a/app/modules/services/task_category_import.py b/app/modules/services/task_category_import.py index 3081233..b888bc6 100644 --- a/app/modules/services/task_category_import.py +++ b/app/modules/services/task_category_import.py @@ -277,6 +277,7 @@ async def task_category_import_submit( if template is None or int(template.tenant_id) != tenant_id: raise ValueError(f"Row {result.row_no}: task template changed during import. Please retry.") template.task_category = result.task_category + template.is_customized = True if update_existing_tasks: existing = db.execute( diff --git a/app/modules/services/templates/services/task_template_detail.html b/app/modules/services/templates/services/task_template_detail.html index 938e9f4..f909e1d 100644 --- a/app/modules/services/templates/services/task_template_detail.html +++ b/app/modules/services/templates/services/task_template_detail.html @@ -25,6 +25,9 @@ {% if request.query_params.get('move_error') %}
Task could not be moved.
{% endif %} {% if request.query_params.get('imported') %}
Excel import completed: {{ request.query_params.get('created','0') }} created, {{ request.query_params.get('updated','0') }} updated.
{% endif %} {% if request.query_params.get('import_error') %}
Excel import failed validation. No partial update was committed.
{% endif %} + {% if request.query_params.get('system_upgraded') %}
System update applied to the firm task. Safe unstarted tasks in open engagements were refreshed automatically.
{% endif %} + {% if request.query_params.get('system_kept') %}
Firm customization retained for this system revision. You will be prompted again when the system default changes in a later revision.
{% endif %} + {% if request.query_params.get('sync_success') %}
System defaults synchronized: {{ request.query_params.get('created','0') }} created, {{ request.query_params.get('updated','0') }} updated, {{ request.query_params.get('custom_updates','0') }} customized task(s) awaiting review.
{% endif %} {% if can_manage_tasks %} @@ -46,7 +49,7 @@
System Defaults
{{ default_tasks|length }}
-

Defaults may be copied and customised for the firm.

+

Inherited defaults stay synchronized automatically; customized tasks require Firm Admin approval before a later system update is applied.

@@ -60,7 +63,7 @@ {% if default_tasks|length > 0 %}
- +
{% endif %} @@ -205,6 +208,40 @@
{{ task.task_name }}
+
+ {% if task.source_system_task_id %} + {% if task.is_customized %}Customized{% else %}System inherited{% endif %} + {% else %} + Firm-only + {% endif %} + {% if task.system_update_available %}System update available{% endif %} +
+ {% if task.system_update_available %} + {% set changes = system_update_diff_map.get(task.id, []) if system_update_diff_map else [] %} +
+
A newer system default is available. Your firm customization has not been overwritten.
+ {% if changes %} +
+ {% for change in changes[:5] %} +
{{ change.label }}: {{ change.firm if change.firm is not none else '-' }} → {{ change.system if change.system is not none else '-' }}
+ {% endfor %} + {% if changes|length > 5 %}
+ {{ changes|length - 5 }} more change(s)
{% endif %} +
+ {% endif %} + {% if can_manage_tasks %} +
+
+ + +
+
+ + +
+
+ {% endif %} +
+ {% endif %} {% if task.task_category %}
Category: {{ task.task_category }}
{% endif %} {% if task.description %}
{{ task.description }}
{% endif %} diff --git a/app/modules/services/ui.py b/app/modules/services/ui.py index 0478747..f780c98 100644 --- a/app/modules/services/ui.py +++ b/app/modules/services/ui.py @@ -27,6 +27,14 @@ from app.modules.services.bulk_imports import ( from app.modules.services.catalogue_export import build_service_catalogue_export from app.modules.services.default_tasks_export import build_default_tasks_export from app.modules.services.firm_tasks_export import build_firm_tasks_export +from app.modules.services.default_task_sync import ( + accept_system_update_for_firm_task, + keep_firm_customization_for_system_revision, + mark_firm_task_customized, + sync_firm_tasks_from_system_defaults, + sync_system_defaults_to_all_firms, + system_task_diff, +) from app.modules.services.task_categories_excel import build_task_categories_export, import_task_categories from app.modules.services.task_tools import list_task_tools, normalize_task_tool_code from app.modules.services.task_duplicates import find_duplicate_pairs, merge_default_tasks, merge_firm_tasks @@ -944,51 +952,21 @@ def _copy_system_defaults_if_firm_tasks_empty( catalogue_id: int, user_id: int, ) -> tuple[int, bool]: - """Initialise firm task templates once without overwriting custom firm tasks.""" + """Initialise/synchronise inherited firm tasks from the current system defaults.""" existing_tasks = get_firm_task_templates( db, tenant_id=tenant_id, catalogue_id=catalogue_id, ) - if existing_tasks: - return 0, True - - defaults = get_default_task_templates(db, catalogue_id=catalogue_id) - copied_count = 0 - for default in defaults: - db.add(FirmServiceTaskTemplate( - tenant_id=tenant_id, - service_catalogue_id=catalogue_id, - task_name=default.task_name, - description=default.description, - sequence_no=default.sequence_no, - default_role_name=default.default_role_name, - eligible_role_names=getattr(default, "eligible_role_names", None) or default.default_role_name, - is_mandatory=default.is_mandatory, - requires_review=default.requires_review, - normal_review_role=getattr(default, "normal_review_role", None), - task_category_id=(firm_category.id if (firm_category := ensure_task_category(db, catalogue_id=catalogue_id, tenant_id=tenant_id, name=getattr(default, "task_category", None), user_id=user_id)) else None), - task_category=(firm_category.name if firm_category else None), - response_required=getattr(default, "response_required", False), - response_type=getattr(default, "response_type", "NONE") or "NONE", - evidence_required=getattr(default, "evidence_required", False), - remarks_required_if_no=getattr(default, "remarks_required_if_no", False), - task_tool_code=getattr(default, "task_tool_code", "NONE") or "NONE", - is_aqmm_task=getattr(default, "is_aqmm_task", False), - aqmm_mandatory=getattr(default, "aqmm_mandatory", False), - aqmm_evidence_required=getattr(default, "aqmm_evidence_required", False), - aqmm_manager_review_required=getattr(default, "aqmm_manager_review_required", False), - aqmm_partner_review_required=getattr(default, "aqmm_partner_review_required", False), - aqmm_review_partner_required=getattr(default, "aqmm_review_partner_required", False), - aqmm_blocks_final_release=getattr(default, "aqmm_blocks_final_release", False), - aqmm_reference=getattr(default, "aqmm_reference", None), - is_active=default.is_active, - created_by_user_id=user_id, - updated_by_user_id=user_id, - )) - copied_count += 1 - - return copied_count, False + had_existing = bool(existing_tasks) + result = sync_firm_tasks_from_system_defaults( + db, + tenant_id=tenant_id, + service_catalogue_id=catalogue_id, + updated_by_user_id=user_id, + sync_open_engagements=False, + ) + return result.created, had_existing @router.post('/catalogue/bulk-selection') @@ -1239,6 +1217,12 @@ def task_templates_detail(request: Request, catalogue_id: int): if not selection or not selection.is_enabled: return RedirectResponse(url='/services', status_code=303) tasks = get_firm_task_templates(db, tenant_id=tenant_id, catalogue_id=catalogue_id) + system_update_diff_map = {} + for firm_task in tasks: + if getattr(firm_task, "system_update_available", False) and getattr(firm_task, "source_system_task_id", None): + system_task = db.get(ServiceDefaultTaskTemplate, int(firm_task.source_system_task_id)) + if system_task is not None: + system_update_diff_map[firm_task.id] = system_task_diff(system_task, firm_task) task_requirement_map = { task.id: list_task_document_requirements(db, tenant_id=tenant_id, firm_task_template_id=task.id) for task in tasks @@ -1258,6 +1242,7 @@ def task_templates_detail(request: Request, catalogue_id: int): task_templates=tasks, task_requirement_map=task_requirement_map, task_template_file_map=task_template_file_map, + system_update_diff_map=system_update_diff_map, can_manage_tasks=_can_manage_firm_tasks(db, user), default_tasks=get_default_task_templates(db, catalogue_id=catalogue_id), is_system_admin=_is_system_admin(db, user), @@ -1311,6 +1296,7 @@ def task_template_create_submit(request: Request, catalogue_id: int, task_name: aqmm_blocks_final_release=aqmm_blocks_final_release is not None, aqmm_reference=aqmm_reference.strip() or None, is_active=is_active is not None, + is_customized=True, created_by_user_id=user.id, updated_by_user_id=user.id, ) @@ -1395,7 +1381,15 @@ def default_template_create_submit(request: Request, catalogue_id: int, task_nam if not catalogue: return RedirectResponse(url='/services/defaults', status_code=303) row = ServiceDefaultTaskTemplate(service_catalogue_id=catalogue_id, task_name=task_name.strip(), description=description.strip() or None, default_role_name=default_role_name.strip() or None, sequence_no=sequence_no or next_default_task_sequence(db, catalogue_id=catalogue_id), is_mandatory=is_mandatory is not None, requires_review=requires_review is not None, normal_review_role=_normal_review_role_value(normal_review_role, requires_review=requires_review is not None), task_category_id=(category.id if (category := ensure_task_category(db, catalogue_id=catalogue_id, tenant_id=None, name=task_category, user_id=user.id)) else None), task_category=(category.name if category else None), response_required=response_required is not None, response_type=(response_type.strip().upper() or "NONE"), task_tool_code=normalize_task_tool_code(task_tool_code), evidence_required=evidence_required is not None, remarks_required_if_no=remarks_required_if_no is not None, is_aqmm_task=is_aqmm_task is not None, aqmm_mandatory=aqmm_mandatory is not None, aqmm_evidence_required=aqmm_evidence_required is not None, aqmm_manager_review_required=aqmm_manager_review_required is not None, aqmm_partner_review_required=aqmm_partner_review_required is not None, aqmm_review_partner_required=aqmm_review_partner_required is not None, aqmm_blocks_final_release=aqmm_blocks_final_release is not None, aqmm_reference=aqmm_reference.strip() or None, is_active=is_active is not None) - db.add(row); db.commit() + db.add(row) + db.flush() + sync_system_defaults_to_all_firms( + db, + service_catalogue_id=catalogue_id, + updated_by_user_id=user.id, + sync_open_engagements=True, + ) + db.commit() return RedirectResponse(url=f'/services/catalogue/{catalogue_id}/defaults', status_code=303) finally: db.close() @@ -1415,46 +1409,22 @@ def copy_defaults_to_firm(request: Request, catalogue_id: int, csrf_token: str = selection = get_firm_selection(db, tenant_id=tenant_id, catalogue_id=catalogue_id) if not selection or not selection.is_enabled: return RedirectResponse(url='/services', status_code=303) - existing_sequences = {t.sequence_no for t in get_firm_task_templates(db, tenant_id=tenant_id, catalogue_id=catalogue_id)} - defaults = get_default_task_templates(db, catalogue_id=catalogue_id) - next_seq = next_task_sequence(db, tenant_id=tenant_id, catalogue_id=catalogue_id) - for d in defaults: - seq = d.sequence_no - if seq in existing_sequences: - seq = next_seq - next_seq += 1 - existing_sequences.add(seq) - db.add(FirmServiceTaskTemplate( - tenant_id=tenant_id, - service_catalogue_id=catalogue_id, - task_name=d.task_name, - description=d.description, - sequence_no=seq, - default_role_name=d.default_role_name, - is_mandatory=d.is_mandatory, - requires_review=d.requires_review, - normal_review_role=getattr(d, "normal_review_role", None), - task_category_id=(firm_category.id if (firm_category := ensure_task_category(db, catalogue_id=catalogue_id, tenant_id=tenant_id, name=getattr(d, "task_category", None), user_id=user.id)) else None), - task_category=(firm_category.name if firm_category else None), - response_required=getattr(d, "response_required", False), - response_type=getattr(d, "response_type", "NONE") or "NONE", - evidence_required=getattr(d, "evidence_required", False), - remarks_required_if_no=getattr(d, "remarks_required_if_no", False), - task_tool_code=getattr(d, "task_tool_code", "NONE") or "NONE", - is_aqmm_task=getattr(d, "is_aqmm_task", False), - aqmm_mandatory=getattr(d, "aqmm_mandatory", False), - aqmm_evidence_required=getattr(d, "aqmm_evidence_required", False), - aqmm_manager_review_required=getattr(d, "aqmm_manager_review_required", False), - aqmm_partner_review_required=getattr(d, "aqmm_partner_review_required", False), - aqmm_review_partner_required=getattr(d, "aqmm_review_partner_required", False), - aqmm_blocks_final_release=getattr(d, "aqmm_blocks_final_release", False), - aqmm_reference=getattr(d, "aqmm_reference", None), - is_active=d.is_active, - created_by_user_id=user.id, - updated_by_user_id=user.id, - )) + result = sync_firm_tasks_from_system_defaults( + db, + tenant_id=tenant_id, + service_catalogue_id=catalogue_id, + updated_by_user_id=user.id, + sync_open_engagements=True, + ) db.commit() - return RedirectResponse(url=f'/services/templates/{catalogue_id}', status_code=303) + return RedirectResponse( + url=( + f'/services/templates/{catalogue_id}?sync_success=1' + f'&created={result.created}&updated={result.updated}' + f'&custom_updates={result.custom_updates_available}' + ), + status_code=303, + ) finally: db.close() @@ -1631,6 +1601,7 @@ def firm_task_template_toggle_active( ) task.is_active = not bool(task.is_active) + mark_firm_task_customized(task) task.updated_by_user_id = user.id db.commit() @@ -1643,6 +1614,70 @@ def firm_task_template_toggle_active( db.close() +@router.post('/templates/{catalogue_id}/tasks/{task_id}/system-update/upgrade') +def firm_task_accept_system_update( + request: Request, + catalogue_id: int, + task_id: int, + csrf_token: str = Form(...), +): + validate_csrf(request, csrf_token) + db = CommonSessionLocal() + try: + user = get_current_user(request, db=db) + if not user: + return RedirectResponse(url='/login', status_code=303) + if not _can_manage_firm_tasks(db, user): + return _redirect_denied() + tenant_id = _active_tenant_id(request, user) + task = get_firm_task_template(db, tenant_id=tenant_id, catalogue_id=catalogue_id, task_id=task_id) + if not task or not getattr(task, 'system_update_available', False): + return RedirectResponse(url=f'/services/templates/{catalogue_id}?error=system_update_missing', status_code=303) + result = accept_system_update_for_firm_task(db, task=task, user_id=user.id, sync_open_engagements=True) + db.commit() + return RedirectResponse( + url=(f'/services/templates/{catalogue_id}?system_upgraded=1' + f'&engagement_created={result.engagement_created}' + f'&engagement_updated={result.engagement_updated_pending}' + f'&engagement_deactivated={result.engagement_deactivated_pending}'), + status_code=303, + ) + except ValueError: + db.rollback() + return RedirectResponse(url=f'/services/templates/{catalogue_id}?error=system_update_missing', status_code=303) + finally: + db.close() + + +@router.post('/templates/{catalogue_id}/tasks/{task_id}/system-update/keep-firm') +def firm_task_keep_customization( + request: Request, + catalogue_id: int, + task_id: int, + csrf_token: str = Form(...), +): + validate_csrf(request, csrf_token) + db = CommonSessionLocal() + try: + user = get_current_user(request, db=db) + if not user: + return RedirectResponse(url='/login', status_code=303) + if not _can_manage_firm_tasks(db, user): + return _redirect_denied() + tenant_id = _active_tenant_id(request, user) + task = get_firm_task_template(db, tenant_id=tenant_id, catalogue_id=catalogue_id, task_id=task_id) + if not task or not getattr(task, 'system_update_available', False): + return RedirectResponse(url=f'/services/templates/{catalogue_id}?error=system_update_missing', status_code=303) + keep_firm_customization_for_system_revision(db, task=task, user_id=user.id) + db.commit() + return RedirectResponse(url=f'/services/templates/{catalogue_id}?system_kept=1', status_code=303) + except ValueError: + db.rollback() + return RedirectResponse(url=f'/services/templates/{catalogue_id}?error=system_update_missing', status_code=303) + finally: + db.close() + + @router.post('/templates/{catalogue_id}/sync-open-engagements') def sync_firm_tasks_to_open_engagements( request: Request, @@ -1728,6 +1763,8 @@ def firm_task_template_move( # Swap safely across the existing unique tenant/service/sequence constraint. min_seq = min(int(row.sequence_no) for row in tasks) + mark_firm_task_customized(current) + mark_firm_task_customized(other) current.sequence_no = min_seq - 1000000 - int(current.id) current.updated_by_user_id = user.id db.flush() @@ -1865,6 +1902,7 @@ def firm_task_template_edit_submit( task.aqmm_blocks_final_release = aqmm_blocks_final_release is not None task.aqmm_reference = aqmm_reference.strip() or None task.is_active = is_active is not None + mark_firm_task_customized(task) task.updated_by_user_id = user.id sync_result = None @@ -2015,6 +2053,13 @@ def default_task_template_edit_submit( task.aqmm_reference = aqmm_reference.strip() or None task.is_active = is_active is not None + db.flush() + sync_system_defaults_to_all_firms( + db, + service_catalogue_id=catalogue_id, + updated_by_user_id=user.id, + sync_open_engagements=True, + ) db.commit() return RedirectResponse(url=f'/services/catalogue/{catalogue_id}/defaults', status_code=303) finally: @@ -2102,7 +2147,9 @@ def default_task_duplicates_merge(request: Request,catalogue_id:int,master_task_ if not user: return RedirectResponse(url='/login',status_code=303) require_permission(db,user,'service_tasks.edit') if not _is_system_admin(db,user): return _redirect_denied() - count=merge_default_tasks(db,catalogue_id=catalogue_id,master_task_id=master_task_id,source_task_ids=source_task_ids); db.commit() + count=merge_default_tasks(db,catalogue_id=catalogue_id,master_task_id=master_task_id,source_task_ids=source_task_ids) + sync_system_defaults_to_all_firms(db,service_catalogue_id=catalogue_id,updated_by_user_id=user.id,sync_open_engagements=True) + db.commit() return RedirectResponse(url=f'/services/catalogue/{catalogue_id}/defaults/duplicates?merged={count}',status_code=303) except ValueError: db.rollback(); return RedirectResponse(url=f'/services/catalogue/{catalogue_id}/defaults/duplicates?merge_error=1',status_code=303) @@ -2237,9 +2284,11 @@ def firm_task_category_update(request: Request, catalogue_id: int, category_id: duplicate=db.execute(select(ServiceTaskCategory.id).where(ServiceTaskCategory.tenant_id==tenant_id, ServiceTaskCategory.service_catalogue_id==catalogue_id, ServiceTaskCategory.code==clean_code, ServiceTaskCategory.id!=row.id)).scalar_one_or_none() if duplicate is not None: return RedirectResponse(url=f'/services/templates/{catalogue_id}/categories?error=duplicate_code', status_code=303) old_name=row.name; row.name=clean_name; row.code=clean_code; row.sort_order=sort_order; row.is_active=is_active is not None; row.updated_by_user_id=user.id - for task in db.execute(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.tenant_id==tenant_id, FirmServiceTaskTemplate.service_catalogue_id==catalogue_id, FirmServiceTaskTemplate.task_category_id==row.id)).scalars().all(): task.task_category=clean_name + for task in db.execute(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.tenant_id==tenant_id, FirmServiceTaskTemplate.service_catalogue_id==catalogue_id, FirmServiceTaskTemplate.task_category_id==row.id)).scalars().all(): + task.task_category=clean_name; mark_firm_task_customized(task) # Backward-compatible rows created before category IDs are also renamed when their text exactly matches. - for task in db.execute(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.tenant_id==tenant_id, FirmServiceTaskTemplate.service_catalogue_id==catalogue_id, FirmServiceTaskTemplate.task_category_id.is_(None), FirmServiceTaskTemplate.task_category==old_name)).scalars().all(): task.task_category_id=row.id; task.task_category=clean_name + for task in db.execute(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.tenant_id==tenant_id, FirmServiceTaskTemplate.service_catalogue_id==catalogue_id, FirmServiceTaskTemplate.task_category_id.is_(None), FirmServiceTaskTemplate.task_category==old_name)).scalars().all(): + task.task_category_id=row.id; task.task_category=clean_name; mark_firm_task_customized(task) db.commit(); return RedirectResponse(url=f'/services/templates/{catalogue_id}/categories?updated=1', status_code=303) finally: db.close() @@ -2334,6 +2383,7 @@ def system_task_category_update(request: Request, catalogue_id: int, category_id old_name=row.name; row.name=clean_name; row.code=clean_code; row.sort_order=sort_order; row.is_active=is_active is not None; row.updated_by_user_id=user.id for task in db.execute(select(ServiceDefaultTaskTemplate).where(ServiceDefaultTaskTemplate.service_catalogue_id==catalogue_id, ServiceDefaultTaskTemplate.task_category_id==row.id)).scalars().all(): task.task_category=row.name for task in db.execute(select(ServiceDefaultTaskTemplate).where(ServiceDefaultTaskTemplate.service_catalogue_id==catalogue_id, ServiceDefaultTaskTemplate.task_category_id.is_(None), ServiceDefaultTaskTemplate.task_category==old_name)).scalars().all(): task.task_category_id=row.id; task.task_category=row.name + sync_system_defaults_to_all_firms(db,service_catalogue_id=catalogue_id,updated_by_user_id=user.id,sync_open_engagements=True) db.commit(); return RedirectResponse(url=f'/services/catalogue/{catalogue_id}/task-categories?updated=1',status_code=303) finally: db.close() @@ -2528,8 +2578,6 @@ def upload_firm_task_templates(request: Request, file: UploadFile = File(...), u @router.post("/catalogue/{catalogue_id}/firm-tasks/sync-system-defaults") def sync_system_default_tasks_for_firm(request: Request, catalogue_id: int): """Idempotent replacement for the old insert-only 'Copy System Default Tasks' action.""" - from app.modules.services.default_task_sync import sync_firm_tasks_from_system_defaults - db = CommonSessionLocal() try: current_user = get_current_user(request, db=db)