Fix retirement propagation and task counts

app/modules/services/default_task_sync.py

@@ -109,6 +109,29 @@ def firm_task_hash(target: FirmServiceTaskTemplate) -> str:
     return hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
 
+def _legacy_retirement_equivalent(
+    source: ServiceDefaultTaskTemplate,
+    target: FirmServiceTaskTemplate,
+) -> bool:
+    """Return True when a legacy firm row differs only because the system row was retired.
+
+    Full-sync retirement moves omitted system rows into a high temporary sequence range
+    before setting is_active=False. Legacy firm rows created before provenance tracking still
+    carry their old active flag and old sequence. Those two differences must not make an
+    otherwise inherited task look like a deliberate firm customization.
+
+    Any substantive field difference still protects the firm row as customized.
+    """
+    source_snapshot = _snapshot_dict(source)
+    target_snapshot = _snapshot_dict(target)
+    source_snapshot.pop("is_active", None)
+    target_snapshot.pop("is_active", None)
+    if int(getattr(source, "sequence_no", 0) or 0) >= 100000:
+        source_snapshot.pop("sequence_no", None)
+        target_snapshot.pop("sequence_no", None)
+    return source_snapshot == target_snapshot
+
+
 def system_task_diff(source: ServiceDefaultTaskTemplate, target: FirmServiceTaskTemplate) -> list[dict[str, Any]]:
     labels = {
         "task_name": "Task name",
@@ -367,11 +390,31 @@ def sync_firm_tasks_from_system_defaults(
             # For legacy rows with no inheritance history, infer whether they were
             # already firm-customized by comparing the full portable snapshot.
             if linked_from_legacy and target.last_synced_system_hash is None:
+                if not source.is_active and _legacy_retirement_equivalent(source, target):
+                    # This was an inherited legacy row whose system source has just been
+                    # retired by Full Synchronization. Do not misclassify the active/old
+                    # sequence difference as a firm customization; allow normal retirement
+                    # propagation below.
+                    target.is_customized = False
+                else:
                     target.is_customized = firm_task_hash(target) != latest_hash
                 if not target.is_customized:
                     target.last_synced_system_hash = latest_hash
                     target.last_reviewed_system_hash = latest_hash
 
+            # Repair rows linked by an earlier rollout where retirement-only differences
+            # were incorrectly interpreted as customization. This is deliberately limited
+            # to rows that have never had a successful inherited snapshot recorded.
+            if (
+                not source.is_active
+                and target.is_customized
+                and target.last_synced_system_hash is None
+                and _legacy_retirement_equivalent(source, target)
+            ):
+                target.is_customized = False
+                target.system_update_available = False
+                target.system_update_detected_at_utc = None
+
             # Detect out-of-band edits to a linked inherited row as customization.
             if (
                 not target.is_customized

app/modules/services/templates/services/default_templates_detail.html

@@ -11,14 +11,21 @@
 
   {% if request.query_params.get('imported') %}<div class="rounded-xl border border-emerald-200 bg-emerald-50 p-4 text-sm text-emerald-800">Excel import completed: {{ request.query_params.get('created','0') }} created, {{ request.query_params.get('updated','0') }} updated, {{ request.query_params.get('retired','0') }} retired.{% if request.query_params.get('full_sync') == '1' %} Full Synchronization was applied.{% endif %}</div>{% endif %}
   {% if request.query_params.get('import_error') %}<div class="rounded-xl border border-red-200 bg-red-50 p-4 text-sm text-red-800">Excel import failed validation. No partial update was committed. Review the workbook and try again.</div>{% endif %}
+  <section class="grid gap-4 md:grid-cols-2">
+    <div class="rounded-2xl bg-white p-5 shadow-soft"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Active System Defaults</div><div class="mt-2 text-2xl font-semibold text-slate-900">{{ active_default_task_count }}</div><p class="mt-1 text-sm text-slate-500">These tasks are available for synchronization to firms and new engagement work.</p></div>
+    <div class="rounded-2xl bg-white p-5 shadow-soft"><div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Retired System Defaults</div><div class="mt-2 text-2xl font-semibold text-slate-900">{{ retired_default_task_count }}</div><p class="mt-1 text-sm text-slate-500">Soft-retired tasks remain in the database so historical engagement and audit records are preserved.</p></div>
+  </section>
   <section class="rounded-2xl border border-emerald-200 bg-emerald-50 p-4">
     <div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between"><div><h3 class="text-sm font-semibold text-emerald-950">Export · Edit · Re-import This Service</h3><p class="mt-1 text-xs text-emerald-800">Export this service's current tasks, edit the Import Ready sheet, then re-import. Existing exports now carry a stable system_task_id so renaming and resequencing update the correct task.</p></div>
     <form method="post" action="/services/catalogue/{{ service.id }}/defaults/import" enctype="multipart/form-data" class="flex flex-wrap items-center gap-2" onsubmit="if (this.full_sync.checked) return confirm('Full Synchronization will retire existing system-default tasks for this service that are missing from the workbook. Historical work will be preserved. Continue?');"><input type="hidden" name="csrf_token" value="{{ csrf_token }}"><input type="file" name="file" accept=".xlsx" required class="max-w-xs rounded-xl border border-emerald-200 bg-white px-3 py-2 text-xs"><label class="inline-flex items-center gap-2 rounded-xl border border-amber-200 bg-amber-50 px-3 py-2 text-xs font-medium text-amber-900"><input type="checkbox" name="full_sync"> Full Synchronization (retire missing tasks)</label><button class="rounded-xl bg-emerald-700 px-4 py-2 text-sm font-semibold text-white hover:bg-emerald-800">Import Updated Excel</button></form></div>
   </section>
   <section class="rounded-2xl bg-white p-5 shadow-soft">
+    <div class="mb-4 flex items-center justify-between gap-3">
+      <div><h3 class="text-sm font-semibold text-slate-900">System Default Task List</h3><p class="text-xs text-slate-500">Active tasks are shown by default. Retired tasks remain available for audit/history review.</p></div>
+      <a href="{% if show_retired %}/services/catalogue/{{ service.id }}/defaults{% else %}/services/catalogue/{{ service.id }}/defaults?show_retired=1{% endif %}" class="rounded-lg border border-slate-300 bg-white px-3 py-1.5 text-xs font-semibold text-slate-700 hover:bg-slate-50">{% if show_retired %}Hide Retired{% else %}Show Retired ({{ retired_default_task_count }}){% endif %}</a>
+    </div>
     <div class="space-y-3">
-      {% for task in default_tasks %}
-      {% if task.is_active or is_system_admin %}
+      {% for task in display_default_tasks %}
       <div class="rounded-xl border border-slate-200 p-4">
   <div class="flex items-center justify-between gap-4">
     <div>
@@ -40,8 +47,7 @@
   <p class="mt-2 text-sm text-slate-700">{{ task.description }}</p>
   {% endif %}
 </div>
-      {% endif %}
-      {% else %}<div class="rounded-xl border border-dashed border-slate-300 p-6 text-sm text-slate-500">No default tasks configured yet.</div>{% endfor %}
+      {% else %}<div class="rounded-xl border border-dashed border-slate-300 p-6 text-sm text-slate-500">No active default tasks configured. Use Show Retired to view inactive historical defaults.</div>{% endfor %}
     </div>
   </section>
   <section class="rounded-2xl bg-white p-5 shadow-soft">

app/modules/services/templates/services/task_template_detail.html

@@ -43,13 +43,13 @@
     </div>
     <div class="rounded-2xl bg-white p-5 shadow-soft">
       <div class="text-xs font-semibold uppercase tracking-wide text-slate-500">Firm Tasks</div>
-      <div class="mt-2 text-lg font-semibold text-slate-900">{{ task_templates|length }}</div>
-      <p class="mt-1 text-sm text-slate-500">Tasks customised for the active firm.</p>
+      <div class="mt-2 text-lg font-semibold text-slate-900">{{ active_task_count }}</div>
+      <p class="mt-1 text-sm text-slate-500">Active tasks for the firm. {{ retired_task_count }} retired task(s) are preserved for history.</p>
     </div>
     <div class="rounded-2xl bg-white p-5 shadow-soft">
       <div class="text-xs font-semibold uppercase tracking-wide text-slate-500">System Defaults</div>
-      <div class="mt-2 text-lg font-semibold text-slate-900">{{ default_tasks|length }}</div>
-      <p class="mt-1 text-sm text-slate-500">Inherited defaults stay synchronized automatically; customized tasks require Firm Admin approval before a later system update is applied.</p>
+      <div class="mt-2 text-lg font-semibold text-slate-900">{{ active_default_task_count }}</div>
+      <p class="mt-1 text-sm text-slate-500">Active inherited system defaults. {{ retired_default_task_count }} retired default(s) are preserved for history; customized tasks require Firm Admin approval before a later system update is applied.</p>
     </div>
   </section>
 
@@ -60,7 +60,7 @@
         <h3 class="text-sm font-semibold text-slate-900">Add Firm Task</h3>
         <p class="text-sm text-slate-500">Create a task template for this firm and service.</p>
       </div>
-      {% if default_tasks|length > 0 %}
+      {% if active_default_task_count > 0 %}
       <form method="post" action="/services/catalogue/{{ service.id }}/firm-tasks/sync-system-defaults">
         <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
         <button class="rounded-xl border border-brand-300 px-4 py-2 text-sm font-medium text-brand-700 hover:bg-brand-50" type="submit">Sync System Defaults</button>
@@ -164,9 +164,12 @@
   {% endif %}
 
   <section class="overflow-hidden rounded-2xl bg-white shadow-soft">
-    <div class="border-b border-slate-100 px-5 py-4">
+    <div class="flex items-center justify-between gap-3 border-b border-slate-100 px-5 py-4">
+      <div>
         <h3 class="text-sm font-semibold text-slate-900">Firm Task List</h3>
-      <p class="text-sm text-slate-500">These tasks will be used when work is generated for this firm.</p>
+        <p class="text-sm text-slate-500">Active tasks are used for new work. Retired tasks remain stored for historical references.</p>
+      </div>
+      <a href="{% if show_retired %}/services/templates/{{ service.id }}{% else %}/services/templates/{{ service.id }}?show_retired=1{% endif %}" class="rounded-lg border border-slate-300 bg-white px-3 py-1.5 text-xs font-semibold text-slate-700 hover:bg-slate-50">{% if show_retired %}Hide Retired{% else %}Show Retired ({{ retired_task_count }}){% endif %}</a>
     </div>
     <table class="min-w-full divide-y divide-slate-200">
       <thead class="bg-slate-50">
@@ -181,7 +184,7 @@
         </tr>
       </thead>
       <tbody class="divide-y divide-slate-100">
-        {% for task in task_templates %}
+        {% for task in display_task_templates %}
         <tr>
           <td class="px-4 py-3 text-sm font-medium text-slate-900 align-top">
             <div class="flex items-center gap-2">
@@ -219,7 +222,8 @@
             {% if task.system_update_available %}
               {% set changes = system_update_diff_map.get(task.id, []) if system_update_diff_map else [] %}
               <div class="mt-2 rounded-xl border border-violet-200 bg-violet-50 p-3 text-xs text-violet-950">
-                <div class="font-semibold">A newer system default is available. Your firm customization has not been overwritten.</div>
+                {% set retirement_update = system_update_retirement_map.get(task.id, false) if system_update_retirement_map else false %}
+                <div class="font-semibold">{% if retirement_update %}This system default has been retired. Your customized firm task has not been disabled automatically.{% else %}A newer system default is available. Your firm customization has not been overwritten.{% endif %}</div>
                 {% if changes %}
                 <div class="mt-2 space-y-1">
                   {% for change in changes[:5] %}
@@ -230,13 +234,13 @@
                 {% endif %}
                 {% if can_manage_tasks %}
                 <div class="mt-3 flex flex-wrap gap-2">
-                  <form method="post" action="/services/templates/{{ service.id }}/tasks/{{ task.id }}/system-update/upgrade" onsubmit="return confirm('Upgrade this customized firm task to the latest system default? Safe unstarted tasks in open engagements will also be refreshed.');">
+                  <form method="post" action="/services/templates/{{ service.id }}/tasks/{{ task.id }}/system-update/upgrade" onsubmit="return confirm('{% if retirement_update %}Retire this firm task in line with the system default? Safe pending/unstarted tasks in open engagements will also be deactivated; historical work is preserved.{% else %}Upgrade this customized firm task to the latest system default? Safe unstarted tasks in open engagements will also be refreshed.{% endif %}');">
                     <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-                    <button class="rounded-lg bg-violet-700 px-3 py-1.5 text-[11px] font-semibold text-white hover:bg-violet-800">Upgrade to System</button>
+                    <button class="rounded-lg bg-violet-700 px-3 py-1.5 text-[11px] font-semibold text-white hover:bg-violet-800">{% if retirement_update %}Retire with System{% else %}Upgrade to System{% endif %}</button>
                   </form>
                   <form method="post" action="/services/templates/{{ service.id }}/tasks/{{ task.id }}/system-update/keep-firm">
                     <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
-                    <button class="rounded-lg border border-violet-300 bg-white px-3 py-1.5 text-[11px] font-semibold text-violet-800 hover:bg-violet-100">Keep Firm Version</button>
+                    <button class="rounded-lg border border-violet-300 bg-white px-3 py-1.5 text-[11px] font-semibold text-violet-800 hover:bg-violet-100">{% if retirement_update %}Keep Firm Task{% else %}Keep Firm Version{% endif %}</button>
                   </form>
                 </div>
                 {% endif %}
@@ -326,7 +330,7 @@
           </td>
         </tr>
         {% else %}
-        <tr><td colspan="7" class="px-4 py-8 text-center text-sm text-slate-500">No firm task templates yet. Copy defaults or add tasks manually.</td></tr>
+        <tr><td colspan="7" class="px-4 py-8 text-center text-sm text-slate-500">No active firm task templates yet. Use Show Retired to view historical inactive tasks, or sync/add tasks as needed.</td></tr>
         {% endfor %}
       </tbody>
     </table>

app/modules/services/ui.py

@@ -1194,7 +1194,7 @@ def task_templates_list(request: Request):
         tenant_id = _active_tenant_id(request, user)
         enabled_rows = list_firm_services_payload(db, tenant_id=tenant_id)
         for row in enabled_rows:
-            row['task_count'] = len(get_firm_task_templates(db, tenant_id=tenant_id, catalogue_id=row['catalogue'].id))
+            row['task_count'] = sum(1 for task in get_firm_task_templates(db, tenant_id=tenant_id, catalogue_id=row['catalogue'].id) if task.is_active)
         return _render(request, 'modules/services/templates/services/task_templates_list.html', db, user, title='Firm Service Task Templates', rows=enabled_rows, can_manage_tasks=_can_manage_firm_tasks(db, user))
     finally:
         db.close()
@@ -1218,11 +1218,13 @@ def task_templates_detail(request: Request, catalogue_id: int):
             return RedirectResponse(url='/services', status_code=303)
         tasks = get_firm_task_templates(db, tenant_id=tenant_id, catalogue_id=catalogue_id)
         system_update_diff_map = {}
+        system_update_retirement_map = {}
         for firm_task in tasks:
             if getattr(firm_task, "system_update_available", False) and getattr(firm_task, "source_system_task_id", None):
                 system_task = db.get(ServiceDefaultTaskTemplate, int(firm_task.source_system_task_id))
                 if system_task is not None:
                     system_update_diff_map[firm_task.id] = system_task_diff(system_task, firm_task)
+                    system_update_retirement_map[firm_task.id] = not bool(system_task.is_active)
         task_requirement_map = {
             task.id: list_task_document_requirements(db, tenant_id=tenant_id, firm_task_template_id=task.id)
             for task in tasks
@@ -1231,6 +1233,9 @@ def task_templates_detail(request: Request, catalogue_id: int):
             task.id: list_task_document_templates(db, tenant_id=tenant_id, firm_task_template_id=task.id)
             for task in tasks
         }
+        default_tasks = get_default_task_templates(db, catalogue_id=catalogue_id)
+        show_retired = request.query_params.get("show_retired") == "1"
+        display_tasks = tasks if show_retired else [task for task in tasks if task.is_active]
         return _render(
             request,
             'modules/services/templates/services/task_template_detail.html',
@@ -1240,11 +1245,20 @@ def task_templates_detail(request: Request, catalogue_id: int):
             service=catalogue,
             selection=selection,
             task_templates=tasks,
+            display_task_templates=display_tasks,
+            show_retired=show_retired,
+            active_task_templates=[task for task in tasks if task.is_active],
+            retired_task_templates=[task for task in tasks if not task.is_active],
+            active_task_count=sum(1 for task in tasks if task.is_active),
+            retired_task_count=sum(1 for task in tasks if not task.is_active),
             task_requirement_map=task_requirement_map,
             task_template_file_map=task_template_file_map,
             system_update_diff_map=system_update_diff_map,
+            system_update_retirement_map=system_update_retirement_map,
             can_manage_tasks=_can_manage_firm_tasks(db, user),
-            default_tasks=get_default_task_templates(db, catalogue_id=catalogue_id),
+            default_tasks=default_tasks,
+            active_default_task_count=sum(1 for task in default_tasks if task.is_active),
+            retired_default_task_count=sum(1 for task in default_tasks if not task.is_active),
             is_system_admin=_is_system_admin(db, user),
             task_categories=list_task_categories(db, catalogue_id=catalogue_id, tenant_id=tenant_id, include_inactive=True),
         )
@@ -1361,7 +1375,24 @@ def default_templates_detail(request: Request, catalogue_id: int):
         if not catalogue:
             return RedirectResponse(url='/services/defaults', status_code=303)
         defaults = get_default_task_templates(db, catalogue_id=catalogue_id)
-        return _render(request, 'modules/services/templates/services/default_templates_detail.html', db, user, title=f'Default Tasks - {catalogue.service_name}', service=catalogue, default_tasks=defaults, task_categories=list_task_categories(db, catalogue_id=catalogue_id, tenant_id=None, include_inactive=True))
+        show_retired = request.query_params.get("show_retired") == "1"
+        display_defaults = defaults if show_retired else [task for task in defaults if task.is_active]
+        return _render(
+            request,
+            'modules/services/templates/services/default_templates_detail.html',
+            db,
+            user,
+            title=f'Default Tasks - {catalogue.service_name}',
+            service=catalogue,
+            default_tasks=defaults,
+            display_default_tasks=display_defaults,
+            show_retired=show_retired,
+            active_default_tasks=[task for task in defaults if task.is_active],
+            retired_default_tasks=[task for task in defaults if not task.is_active],
+            active_default_task_count=sum(1 for task in defaults if task.is_active),
+            retired_default_task_count=sum(1 for task in defaults if not task.is_active),
+            task_categories=list_task_categories(db, catalogue_id=catalogue_id, tenant_id=None, include_inactive=True),
+        )
     finally:
         db.close()
 
@@ -2628,6 +2659,7 @@ def sync_system_default_tasks_for_firm(request: Request, catalogue_id: int):
                 "updated": str(result.updated),
                 "unchanged": str(result.unchanged),
                 "duplicates_disabled": str(result.duplicates_disabled),
+                "custom_updates": str(result.custom_updates_available),
             }
         )