Reconcile firm tasks fully with system defaults

app/modules/services/default_task_sync.py

@@ -56,6 +56,7 @@ class FirmDefaultTaskSyncResult:
     created: int = 0
     updated: int = 0
     duplicates_disabled: int = 0
+    retired: int = 0
     unchanged: int = 0
     custom_updates_available: int = 0
     linked_legacy: int = 0
@@ -515,6 +516,239 @@ def sync_firm_tasks_from_system_defaults(
     return result
 
 
+
+def fully_align_firm_tasks_to_system_defaults(
+    db: Session,
+    *,
+    tenant_id: int,
+    service_catalogue_id: int,
+    updated_by_user_id: int,
+    sync_open_engagements: bool = True,
+) -> FirmDefaultTaskSyncResult:
+    """Fully align one firm's system-derived checklist to the current system master.
+
+    This is the bulk/explicit "Upgrade All to System" operation. It is intentionally
+    stronger than the normal background sync:
+
+    * every row already linked to a system task follows that system task, even when an
+      earlier firm customization exists;
+    * linked rows whose system source is retired are retired at firm level;
+    * legacy unlinked rows are linked only by an exact normalized task-name match;
+    * duplicate rows for the same system source are retired;
+    * active system-derived rows receive the exact current system sequence;
+    * genuine firm-only rows (no reliable system match) are preserved. If one occupies
+      a sequence now required by the system master it is moved to the next free sequence;
+    * engagement instances are synchronized once after the template reconciliation,
+      using the existing safe rollout rules so historical/started/reviewed work is not
+      rewritten.
+
+    No task/template row is hard-deleted.
+    """
+    defaults = db.execute(
+        select(ServiceDefaultTaskTemplate)
+        .where(ServiceDefaultTaskTemplate.service_catalogue_id == service_catalogue_id)
+        .order_by(ServiceDefaultTaskTemplate.sequence_no.asc(), ServiceDefaultTaskTemplate.id.asc())
+    ).scalars().all()
+    firm_rows = db.execute(
+        select(FirmServiceTaskTemplate)
+        .where(
+            FirmServiceTaskTemplate.tenant_id == tenant_id,
+            FirmServiceTaskTemplate.service_catalogue_id == service_catalogue_id,
+        )
+        .order_by(FirmServiceTaskTemplate.sequence_no.asc(), FirmServiceTaskTemplate.id.asc())
+    ).scalars().all()
+
+    result = FirmDefaultTaskSyncResult()
+    reference_counts = _reference_counts(db, firm_task_ids=[int(r.id) for r in firm_rows if r.id])
+    defaults_by_id = {int(row.id): row for row in defaults}
+    defaults_by_name: dict[str, list[ServiceDefaultTaskTemplate]] = {}
+    for source in defaults:
+        key = _normalise_name(source.task_name)
+        if key:
+            defaults_by_name.setdefault(key, []).append(source)
+
+    # Capture original firm ordering before temporarily moving rows away from the
+    # unique (tenant, service, sequence) namespace.
+    original_sequence = {int(row.id): int(row.sequence_no or 0) for row in firm_rows if row.id is not None}
+
+    # Safely recover provenance for legacy rows. Exact name is deliberately the only
+    # automatic fallback here: anything ambiguous remains a genuine firm-only row.
+    for row in firm_rows:
+        if row.source_system_task_id is not None:
+            continue
+        matches = defaults_by_name.get(_normalise_name(row.task_name), [])
+        if len(matches) == 1:
+            row.source_system_task_id = matches[0].id
+            result.linked_legacy += 1
+
+    db.flush()
+
+    by_source: dict[int, list[FirmServiceTaskTemplate]] = {}
+    firm_only: list[FirmServiceTaskTemplate] = []
+    for row in firm_rows:
+        source_id = int(row.source_system_task_id) if row.source_system_task_id is not None else None
+        if source_id is not None and source_id in defaults_by_id:
+            by_source.setdefault(source_id, []).append(row)
+        else:
+            # A missing/deleted source becomes firm-only because the FK uses SET NULL;
+            # never guess and retire it without a reliable system identity.
+            if source_id is not None and source_id not in defaults_by_id:
+                row.source_system_task_id = None
+            firm_only.append(row)
+
+    # Free every sequence first. This allows exact system sequencing even when an old
+    # firm row currently occupies the target number. The unique constraint remains valid
+    # because each temporary sequence is unique.
+    highest_existing = max([int(r.sequence_no or 0) for r in firm_rows] + [0])
+    highest_default = max([int(r.sequence_no or 0) for r in defaults] + [0])
+    temporary_base = max(highest_existing, highest_default, 100000) + 1000000
+    for offset, row in enumerate(sorted(firm_rows, key=lambda r: int(r.id or 0)), start=1):
+        row.sequence_no = temporary_base + offset
+    db.flush()
+
+    canonical_by_source: dict[int, FirmServiceTaskTemplate] = {}
+
+    for source in defaults:
+        source_id = int(source.id)
+        latest_hash = system_task_hash(source)
+        candidates = by_source.get(source_id, [])
+
+        if candidates:
+            # Prefer the currently active row, then the row referenced by the most
+            # engagement history, then the oldest id. This avoids switching canonical
+            # identity unnecessarily while still preserving historical references.
+            target = sorted(
+                candidates,
+                key=lambda row: (
+                    -int(bool(row.is_active)),
+                    -reference_counts.get(int(row.id), 0),
+                    int(row.id),
+                ),
+            )[0]
+            canonical_by_source[source_id] = target
+
+            for duplicate in candidates:
+                if duplicate.id == target.id:
+                    continue
+                if duplicate.is_active:
+                    result.retired += 1
+                duplicate.is_active = False
+                duplicate.is_customized = False
+                duplicate.last_synced_system_hash = latest_hash
+                duplicate.last_reviewed_system_hash = latest_hash
+                duplicate.system_update_available = False
+                duplicate.system_update_detected_at_utc = None
+                duplicate.updated_by_user_id = updated_by_user_id
+                result.duplicates_disabled += 1
+        elif source.is_active:
+            # Create only active system defaults. Retired defaults with no firm row do
+            # not need a new historical firm record.
+            target = FirmServiceTaskTemplate(
+                tenant_id=tenant_id,
+                service_catalogue_id=service_catalogue_id,
+                task_name=source.task_name,
+                sequence_no=temporary_base + len(firm_rows) + len(canonical_by_source) + 1,
+                source_system_task_id=source.id,
+                is_customized=False,
+                last_synced_system_hash=latest_hash,
+                last_reviewed_system_hash=latest_hash,
+                system_update_available=False,
+                created_by_user_id=updated_by_user_id,
+                updated_by_user_id=updated_by_user_id,
+            )
+            db.add(target)
+            db.flush()
+            firm_rows.append(target)
+            by_source.setdefault(source_id, []).append(target)
+            canonical_by_source[source_id] = target
+            result.created += 1
+        else:
+            continue
+
+        target = canonical_by_source[source_id]
+        was_active = bool(target.is_active)
+        changed = _copy_default_columns(
+            db,
+            source,
+            target,
+            tenant_id=tenant_id,
+            user_id=updated_by_user_id,
+            allow_sequence_change=False,
+        )
+        target.source_system_task_id = source.id
+        target.is_customized = False
+        target.last_synced_system_hash = latest_hash
+        target.last_reviewed_system_hash = latest_hash
+        target.system_update_available = False
+        target.system_update_detected_at_utc = None
+        target.updated_by_user_id = updated_by_user_id
+        if was_active and not bool(source.is_active):
+            result.retired += 1
+        if changed:
+            result.updated += 1
+        else:
+            result.unchanged += 1
+
+    db.flush()
+
+    # Active system rows now own the exact master sequence numbers.
+    used_sequences: set[int] = set()
+    active_sources = [row for row in defaults if bool(row.is_active)]
+    for source in sorted(active_sources, key=lambda r: (int(r.sequence_no or 0), int(r.id))):
+        target = canonical_by_source.get(int(source.id))
+        if target is None:
+            continue
+        desired = int(source.sequence_no or 0)
+        target.sequence_no = desired
+        used_sequences.add(desired)
+    db.flush()
+
+    # Genuine firm-only active tasks survive the full upgrade. Keep their original
+    # sequence when it is still free; otherwise move them after the system checklist.
+    next_free = max(used_sequences or {0}) + 1
+    active_firm_only = sorted(
+        [row for row in firm_only if bool(row.is_active)],
+        key=lambda row: (original_sequence.get(int(row.id), 0), int(row.id)),
+    )
+    for row in active_firm_only:
+        preferred = original_sequence.get(int(row.id), 0)
+        if preferred > 0 and preferred not in used_sequences:
+            row.sequence_no = preferred
+            used_sequences.add(preferred)
+        else:
+            while next_free in used_sequences:
+                next_free += 1
+            row.sequence_no = next_free
+            used_sequences.add(next_free)
+            next_free += 1
+
+    # Retired/duplicate templates are retained for history but moved outside the live
+    # sequence range so they can never block exact system sequencing.
+    inactive_rows = [row for row in firm_rows if not bool(row.is_active)]
+    retired_base = max(used_sequences or {0}) + 100000
+    for offset, row in enumerate(sorted(inactive_rows, key=lambda r: int(r.id or 0)), start=1):
+        row.sequence_no = retired_base + offset
+
+    db.flush()
+
+    if sync_open_engagements:
+        from app.modules.services.execution import sync_open_engagement_tasks_for_service
+
+        engagement = sync_open_engagement_tasks_for_service(
+            db,
+            tenant_id=tenant_id,
+            catalogue_id=service_catalogue_id,
+            user_id=updated_by_user_id,
+            include_started_open_tasks=False,
+            safe_system_rollout=True,
+        )
+        result.engagement_created = engagement.get("created", 0)
+        result.engagement_updated_pending = engagement.get("updated_pending", 0)
+        result.engagement_deactivated_pending = engagement.get("deactivated_pending", 0)
+        result.engagement_preserved_history = engagement.get("preserved_history", 0)
+
+    return result
+
 def sync_system_defaults_to_all_firms(
     db: Session,
     *,

app/modules/services/templates/services/task_template_detail.html

@@ -68,12 +68,10 @@
           <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
           <button class="rounded-xl border border-brand-300 px-4 py-2 text-sm font-medium text-brand-700 hover:bg-brand-50" type="submit">Sync System Defaults</button>
         </form>
-        {% if system_update_available_count > 0 %}
+        <form method="post" action="/services/templates/{{ service.id }}/system-updates/upgrade-all" onsubmit="return confirm('Fully align this firm service checklist to the current System Default? All system-derived tasks will follow the current system version and sequence; system-retired tasks will be retired; genuine firm-only tasks will remain. Existing historical, started, completed and reviewed engagement work will be preserved.');">
-        <form method="post" action="/services/templates/{{ service.id }}/system-updates/upgrade-all" onsubmit="return confirm('Upgrade all {{ system_update_available_count }} firm task(s) with pending system updates to the latest System Default? Firm-only tasks will remain unchanged. Safe pending/unstarted tasks in open engagements will be synchronized; started, completed and reviewed history will be preserved.');">
           <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-          <button class="rounded-xl bg-violet-700 px-4 py-2 text-sm font-semibold text-white hover:bg-violet-800" type="submit">Upgrade All to System ({{ system_update_available_count }})</button>
+          <button class="rounded-xl bg-violet-700 px-4 py-2 text-sm font-semibold text-white hover:bg-violet-800" type="submit">Full Upgrade to System{% if system_update_available_count > 0 %} ({{ system_update_available_count }} pending){% endif %}</button>
         </form>
-        {% endif %}
       </div>
       {% endif %}
     </div>
@@ -178,11 +176,11 @@
     <div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
       <div>
         <h3 class="text-sm font-semibold text-violet-950">System updates awaiting firm decision</h3>
-        <p class="mt-1 text-xs text-violet-800">{{ system_update_available_count }} customized task(s) have a newer system revision. Use <span class="font-semibold">Upgrade All to System</span> for full alignment, or continue using the existing task-by-task Upgrade / Keep controls below.</p>
+        <p class="mt-1 text-xs text-violet-800">{{ system_update_available_count }} customized task(s) have a newer system revision. Use <span class="font-semibold">Full Upgrade to System</span> to reconcile the complete system-derived checklist, or continue using the existing task-by-task Upgrade / Keep controls below.</p>
       </div>
-      <form method="post" action="/services/templates/{{ service.id }}/system-updates/upgrade-all" onsubmit="return confirm('Upgrade all {{ system_update_available_count }} firm task(s) with pending system updates to the latest System Default? Firm-only tasks will remain unchanged. Safe pending/unstarted tasks in open engagements will be synchronized; started, completed and reviewed history will be preserved.');">
+      <form method="post" action="/services/templates/{{ service.id }}/system-updates/upgrade-all" onsubmit="return confirm('Fully align every system-derived task for this firm/service to the current System Default? Genuine firm-only tasks and protected engagement history will remain unchanged.');">
         <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-        <button class="whitespace-nowrap rounded-xl bg-violet-700 px-4 py-2 text-sm font-semibold text-white hover:bg-violet-800" type="submit">Upgrade All to System ({{ system_update_available_count }})</button>
+        <button class="whitespace-nowrap rounded-xl bg-violet-700 px-4 py-2 text-sm font-semibold text-white hover:bg-violet-800" type="submit">Full Upgrade to System</button>
       </form>
     </div>
   </section>

app/modules/services/ui.py

@@ -29,6 +29,7 @@ from app.modules.services.default_tasks_export import build_default_tasks_export
 from app.modules.services.firm_tasks_export import build_firm_tasks_export
 from app.modules.services.default_task_sync import (
     accept_system_update_for_firm_task,
+    fully_align_firm_tasks_to_system_defaults,
     keep_firm_customization_for_system_revision,
     mark_firm_task_customized,
     sync_firm_tasks_from_system_defaults,
@@ -1652,12 +1653,13 @@ def firm_tasks_accept_all_system_updates(
     catalogue_id: int,
     csrf_token: str = Form(...),
 ):
-    """Upgrade every pending firm-task system revision for the active firm/service.
+    """Fully reconcile this firm's service checklist to the current system master.
 
-    Firm-only tasks are intentionally untouched. Existing per-task upgrade/keep actions
+    Existing task-by-task Upgrade / Keep controls remain unchanged. This explicit bulk
-    remain available. Engagement synchronization runs once after all template updates so
+    action aligns every reliably system-derived firm task, retires rows whose system
-    pending/unstarted open-engagement tasks are refreshed safely without rewriting
+    source is retired, resolves duplicate inherited rows, restores the exact active
-    started/completed/reviewed history.
+    system sequence, and preserves genuine firm-only tasks. Engagement synchronization
+    runs once using the existing safe-history rules.
     """
     validate_csrf(request, csrf_token)
     db = CommonSessionLocal()
@@ -1673,58 +1675,27 @@ def firm_tasks_accept_all_system_updates(
         if not selection or not selection.is_enabled:
             return RedirectResponse(url='/services', status_code=303)
 
-        pending_tasks = db.scalars(
+        result = fully_align_firm_tasks_to_system_defaults(
-            select(FirmServiceTaskTemplate)
-            .where(
-                FirmServiceTaskTemplate.tenant_id == tenant_id,
-                FirmServiceTaskTemplate.service_catalogue_id == catalogue_id,
-                FirmServiceTaskTemplate.system_update_available.is_(True),
-                FirmServiceTaskTemplate.source_system_task_id.is_not(None),
-            )
-            .order_by(FirmServiceTaskTemplate.sequence_no, FirmServiceTaskTemplate.id)
-        ).all()
-
-        if not pending_tasks:
-            return RedirectResponse(
-                url=f'/services/templates/{catalogue_id}?system_bulk_none=1',
-                status_code=303,
-            )
-
-        upgraded = 0
-        retired = 0
-        for task in pending_tasks:
-            source = db.get(ServiceDefaultTaskTemplate, int(task.source_system_task_id))
-            if source is None:
-                continue
-            if not bool(source.is_active):
-                retired += 1
-            accept_system_update_for_firm_task(
-                db,
-                task=task,
-                user_id=user.id,
-                sync_open_engagements=False,
-            )
-            upgraded += 1
-
-        engagement = sync_open_engagement_tasks_for_service(
             db,
             tenant_id=tenant_id,
-            catalogue_id=catalogue_id,
+            service_catalogue_id=catalogue_id,
-            user_id=user.id,
+            updated_by_user_id=user.id,
-            include_started_open_tasks=False,
+            sync_open_engagements=True,
-            safe_system_rollout=True,
         )
         db.commit()
 
+        aligned = result.created + result.updated + result.unchanged
         return RedirectResponse(
             url=(
                 f'/services/templates/{catalogue_id}?system_bulk_upgraded=1'
-                f'&bulk_count={upgraded}'
+                f'&bulk_count={aligned}'
-                f'&bulk_retired={retired}'
+                f'&bulk_retired={result.retired}'
-                f'&engagement_created={engagement.get("created", 0)}'
+                f'&bulk_created={result.created}'
-                f'&engagement_updated={engagement.get("updated_pending", 0)}'
+                f'&bulk_linked_legacy={result.linked_legacy}'
-                f'&engagement_deactivated={engagement.get("deactivated_pending", 0)}'
+                f'&engagement_created={result.engagement_created}'
-                f'&engagement_preserved={engagement.get("preserved_history", 0)}'
+                f'&engagement_updated={result.engagement_updated_pending}'
+                f'&engagement_deactivated={result.engagement_deactivated_pending}'
+                f'&engagement_preserved={result.engagement_preserved_history}'
             ),
             status_code=303,
         )