diff --git a/app/modules/services/catalogue_merge.py b/app/modules/services/catalogue_merge.py new file mode 100644 index 0000000..0c5aff5 --- /dev/null +++ b/app/modules/services/catalogue_merge.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timezone + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from app.modules.services.models import ( + ClientServiceSubscription, + FirmServiceSelection, + FirmServiceTaskTemplate, + FirmTaskDocumentRequirement, + FirmTaskDocumentTemplate, + ServiceCatalogue, + ServiceDefaultTaskTemplate, + ServiceDueDateExtension, + ServiceDueDateRule, +) + + +@dataclass(frozen=True) +class MergePreview: + source: ServiceCatalogue + target: ServiceCatalogue + counts: dict[str, int] + conflicts: dict[str, int] + + +def _normalise(value: str | None) -> str: + return re.sub(r"[^a-z0-9]+", "", (value or "").strip().lower()) + + +def _count(db: Session, model, condition) -> int: + return int(db.scalar(select(func.count(model.id)).where(condition)) or 0) + + +def build_merge_preview(db: Session, *, source_id: int, target_id: int) -> MergePreview: + if source_id == target_id: + raise ValueError("Source and target services must be different.") + + source = db.get(ServiceCatalogue, source_id) + target = db.get(ServiceCatalogue, target_id) + if not source or not target: + raise ValueError("The selected source or target service no longer exists.") + + counts = { + "firm_selections": _count(db, FirmServiceSelection, FirmServiceSelection.service_catalogue_id == source_id), + "system_default_tasks": _count(db, ServiceDefaultTaskTemplate, ServiceDefaultTaskTemplate.service_catalogue_id == source_id), + "firm_task_templates": _count(db, FirmServiceTaskTemplate, FirmServiceTaskTemplate.service_catalogue_id == source_id), + "document_requirements": _count(db, FirmTaskDocumentRequirement, FirmTaskDocumentRequirement.service_catalogue_id == source_id), + "document_templates": _count(db, FirmTaskDocumentTemplate, FirmTaskDocumentTemplate.service_catalogue_id == source_id), + "due_date_rules": _count(db, ServiceDueDateRule, ServiceDueDateRule.service_catalogue_id == source_id), + "due_date_extensions": _count(db, ServiceDueDateExtension, ServiceDueDateExtension.service_catalogue_id == source_id), + "client_subscriptions_preserved": _count(db, ClientServiceSubscription, ClientServiceSubscription.service_catalogue_id == source_id), + } + + source_tenants = set(db.scalars(select(FirmServiceSelection.tenant_id).where(FirmServiceSelection.service_catalogue_id == source_id)).all()) + target_tenants = set(db.scalars(select(FirmServiceSelection.tenant_id).where(FirmServiceSelection.service_catalogue_id == target_id)).all()) + + source_default_names = { + _normalise(name) + for name in db.scalars(select(ServiceDefaultTaskTemplate.task_name).where(ServiceDefaultTaskTemplate.service_catalogue_id == source_id)).all() + } + target_default_names = { + _normalise(name) + for name in db.scalars(select(ServiceDefaultTaskTemplate.task_name).where(ServiceDefaultTaskTemplate.service_catalogue_id == target_id)).all() + } + + source_rules = { + _normalise(name) + for name in db.scalars(select(ServiceDueDateRule.rule_name).where(ServiceDueDateRule.service_catalogue_id == source_id)).all() + } + target_rules = { + _normalise(name) + for name in db.scalars(select(ServiceDueDateRule.rule_name).where(ServiceDueDateRule.service_catalogue_id == target_id)).all() + } + + firm_task_conflicts = 0 + source_firm_tasks = db.scalars(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.service_catalogue_id == source_id)).all() + target_firm_tasks = db.scalars(select(FirmServiceTaskTemplate).where(FirmServiceTaskTemplate.service_catalogue_id == target_id)).all() + target_task_keys = {(row.tenant_id, _normalise(row.task_name)) for row in target_firm_tasks} + for row in source_firm_tasks: + if (row.tenant_id, _normalise(row.task_name)) in target_task_keys: + firm_task_conflicts += 1 + + conflicts = { + "firm_selection_conflicts": len(source_tenants & target_tenants), + "system_default_task_name_conflicts": len(source_default_names & target_default_names), + "firm_task_name_conflicts": firm_task_conflicts, + "due_rule_name_conflicts": len(source_rules & target_rules), + } + return MergePreview(source=source, target=target, counts=counts, conflicts=conflicts) + + +def _merge_firm_selections(db: Session, *, source_id: int, target_id: int, actor_user_id: int) -> int: + """Create/consolidate target selections while retaining source selections for history.""" + processed = 0 + source_rows = db.scalars( + select(FirmServiceSelection) + .where(FirmServiceSelection.service_catalogue_id == source_id) + .order_by(FirmServiceSelection.id.asc()) + ).all() + + for source_row in source_rows: + target_row = db.scalar( + select(FirmServiceSelection).where( + FirmServiceSelection.tenant_id == source_row.tenant_id, + FirmServiceSelection.service_catalogue_id == target_id, + ) + ) + if target_row: + target_row.is_enabled = bool(target_row.is_enabled or source_row.is_enabled) + if target_row.default_branch_id is None and source_row.default_branch_id is not None: + target_row.default_branch_id = source_row.default_branch_id + target_row.updated_by_user_id = actor_user_id + target_row.updated_at_utc = datetime.now(timezone.utc) + else: + db.add( + FirmServiceSelection( + tenant_id=source_row.tenant_id, + service_catalogue_id=target_id, + is_enabled=bool(source_row.is_enabled), + default_branch_id=source_row.default_branch_id, + activated_by_user_id=source_row.activated_by_user_id or actor_user_id, + updated_by_user_id=actor_user_id, + ) + ) + source_row.is_enabled = False + source_row.updated_by_user_id = actor_user_id + source_row.updated_at_utc = datetime.now(timezone.utc) + processed += 1 + db.flush() + return processed + + +def _merge_default_tasks(db: Session, *, source_id: int, target_id: int) -> tuple[int, int]: + target_rows = db.scalars( + select(ServiceDefaultTaskTemplate) + .where(ServiceDefaultTaskTemplate.service_catalogue_id == target_id) + .order_by(ServiceDefaultTaskTemplate.sequence_no.asc(), ServiceDefaultTaskTemplate.id.asc()) + ).all() + target_by_name = {_normalise(row.task_name): row for row in target_rows} + next_sequence = max((row.sequence_no for row in target_rows), default=0) + 1 + + moved = 0 + skipped = 0 + source_rows = db.scalars( + select(ServiceDefaultTaskTemplate) + .where(ServiceDefaultTaskTemplate.service_catalogue_id == source_id) + .order_by(ServiceDefaultTaskTemplate.sequence_no.asc(), ServiceDefaultTaskTemplate.id.asc()) + ).all() + for row in source_rows: + key = _normalise(row.task_name) + if key and key in target_by_name: + skipped += 1 + db.delete(row) + continue + row.service_catalogue_id = target_id + row.sequence_no = next_sequence + next_sequence += 1 + target_by_name[key] = row + moved += 1 + db.flush() + return moved, skipped + + +def _merge_document_requirements(db: Session, *, source_template: FirmServiceTaskTemplate, target_template: FirmServiceTaskTemplate, target_id: int) -> None: + target_names = { + _normalise(row.document_name) + for row in db.scalars( + select(FirmTaskDocumentRequirement).where(FirmTaskDocumentRequirement.firm_task_template_id == target_template.id) + ).all() + } + for row in db.scalars( + select(FirmTaskDocumentRequirement).where(FirmTaskDocumentRequirement.firm_task_template_id == source_template.id) + ).all(): + key = _normalise(row.document_name) + if key and key in target_names: + db.delete(row) + continue + row.firm_task_template_id = target_template.id + row.service_catalogue_id = target_id + target_names.add(key) + + for row in db.scalars( + select(FirmTaskDocumentTemplate).where(FirmTaskDocumentTemplate.firm_task_template_id == source_template.id) + ).all(): + row.firm_task_template_id = target_template.id + row.service_catalogue_id = target_id + + +def _merge_firm_tasks(db: Session, *, source_id: int, target_id: int) -> tuple[int, int]: + target_rows = db.scalars( + select(FirmServiceTaskTemplate) + .where(FirmServiceTaskTemplate.service_catalogue_id == target_id) + .order_by(FirmServiceTaskTemplate.tenant_id.asc(), FirmServiceTaskTemplate.sequence_no.asc()) + ).all() + target_by_key = {(row.tenant_id, _normalise(row.task_name)): row for row in target_rows} + max_sequence_by_tenant: dict[int, int] = {} + for row in target_rows: + max_sequence_by_tenant[row.tenant_id] = max(max_sequence_by_tenant.get(row.tenant_id, 0), row.sequence_no) + + moved = 0 + merged = 0 + source_rows = db.scalars( + select(FirmServiceTaskTemplate) + .where(FirmServiceTaskTemplate.service_catalogue_id == source_id) + .order_by(FirmServiceTaskTemplate.tenant_id.asc(), FirmServiceTaskTemplate.sequence_no.asc(), FirmServiceTaskTemplate.id.asc()) + ).all() + for row in source_rows: + key = (row.tenant_id, _normalise(row.task_name)) + target_row = target_by_key.get(key) + if target_row: + _merge_document_requirements(db, source_template=row, target_template=target_row, target_id=target_id) + db.flush() + db.delete(row) + merged += 1 + continue + next_sequence = max_sequence_by_tenant.get(row.tenant_id, 0) + 1 + max_sequence_by_tenant[row.tenant_id] = next_sequence + row.service_catalogue_id = target_id + row.sequence_no = next_sequence + for requirement in row.document_requirements: + requirement.service_catalogue_id = target_id + for template in row.document_templates: + template.service_catalogue_id = target_id + target_by_key[key] = row + moved += 1 + db.flush() + return moved, merged + + +def _merge_due_rules(db: Session, *, source_id: int, target_id: int) -> tuple[int, int]: + target_rows = db.scalars(select(ServiceDueDateRule).where(ServiceDueDateRule.service_catalogue_id == target_id)).all() + target_by_name = {_normalise(row.rule_name): row for row in target_rows} + moved = 0 + merged = 0 + + source_rows = db.scalars(select(ServiceDueDateRule).where(ServiceDueDateRule.service_catalogue_id == source_id)).all() + for row in source_rows: + target_row = target_by_name.get(_normalise(row.rule_name)) + if target_row: + db.query(ServiceDueDateExtension).filter(ServiceDueDateExtension.due_date_rule_id == row.id).update( + {ServiceDueDateExtension.due_date_rule_id: target_row.id}, synchronize_session=False + ) + db.query(ClientServiceSubscription).filter(ClientServiceSubscription.due_date_rule_id == row.id).update( + {ClientServiceSubscription.due_date_rule_id: target_row.id}, synchronize_session=False + ) + db.delete(row) + merged += 1 + else: + row.service_catalogue_id = target_id + target_by_name[_normalise(row.rule_name)] = row + moved += 1 + + db.query(ServiceDueDateExtension).filter(ServiceDueDateExtension.service_catalogue_id == source_id).update( + {ServiceDueDateExtension.service_catalogue_id: target_id}, synchronize_session=False + ) + db.flush() + return moved, merged + + +def merge_service_catalogues( + db: Session, + *, + source_id: int, + target_id: int, + actor_user_id: int, +) -> dict[str, int | str]: + """Safely consolidate reusable configuration into the target service. + + Existing client subscriptions, execution tasks, invoices and consultant requests are + intentionally retained against the source service to preserve historical records. + The source service is disabled after configuration is moved, so all new work uses + the target service. + """ + preview = build_merge_preview(db, source_id=source_id, target_id=target_id) + source = preview.source + target = preview.target + + selections_processed = _merge_firm_selections( + db, + source_id=source_id, + target_id=target_id, + actor_user_id=actor_user_id, + ) + + default_moved, default_skipped = _merge_default_tasks(db, source_id=source_id, target_id=target_id) + firm_moved, firm_merged = _merge_firm_tasks(db, source_id=source_id, target_id=target_id) + rules_moved, rules_merged = _merge_due_rules(db, source_id=source_id, target_id=target_id) + + source.is_active = False + source.is_client_requestable = False + source.is_consultant_requestable = False + source.updated_by_user_id = actor_user_id + source.updated_at_utc = datetime.now(timezone.utc) + + target.is_active = True + target.updated_by_user_id = actor_user_id + target.updated_at_utc = datetime.now(timezone.utc) + + db.flush() + return { + "source_code": source.service_code, + "target_code": target.service_code, + "firm_selections_processed": selections_processed, + "default_tasks_moved": default_moved, + "default_tasks_deduplicated": default_skipped, + "firm_tasks_moved": firm_moved, + "firm_tasks_merged": firm_merged, + "due_rules_moved": rules_moved, + "due_rules_merged": rules_merged, + "historical_subscriptions_preserved": preview.counts["client_subscriptions_preserved"], + } diff --git a/app/modules/services/templates/services/catalogue_list.html b/app/modules/services/templates/services/catalogue_list.html index c8d71da..72380bd 100644 --- a/app/modules/services/templates/services/catalogue_list.html +++ b/app/modules/services/templates/services/catalogue_list.html @@ -12,7 +12,7 @@ Firm Task Templates {% if can_create %} System Default Tasks - Export Excel + Merge Services Add Catalogue Service {% endif %} diff --git a/app/modules/services/templates/services/catalogue_merge.html b/app/modules/services/templates/services/catalogue_merge.html new file mode 100644 index 0000000..8a03a49 --- /dev/null +++ b/app/modules/services/templates/services/catalogue_merge.html @@ -0,0 +1,104 @@ +{% extends "ui/templates/base/layout.html" %} +{% block content %} +
+
+
+

Merge Service Catalogue

+

Consolidate an older duplicate service into the service that should be used for all future work.

+
+ Back to Catalogue +
+ +
+ The source service will be disabled. Firm selections, default tasks, firm task templates, document requirements, document templates and due-date rules are consolidated into the target. Existing client subscriptions, completed or active execution work, invoices and consultant requests remain linked to the source service to preserve historical records. +
+ + {% if request.query_params.get('error') == 'confirmation' %} +
The confirmation text did not match. No records were changed.
+ {% elif request.query_params.get('error') or error %} +
{{ error or 'The selected merge could not be prepared. No records were changed.' }}
+ {% endif %} + +
+
+ + +
+
+ +
+
+ + {% if preview %} +
+
+

Merge direction

+
+
Source: {{ preview.source.service_code }} — {{ preview.source.service_name }}
+
Target: {{ preview.target.service_code }} — {{ preview.target.service_name }}
+
+
+
+

Conflict handling

+
+
Firm selections
{{ preview.conflicts.firm_selection_conflicts }}
+
Default task names
{{ preview.conflicts.system_default_task_name_conflicts }}
+
Firm task names
{{ preview.conflicts.firm_task_name_conflicts }}
+
Due-rule names
{{ preview.conflicts.due_rule_name_conflicts }}
+
+

Conflicting firm selections are consolidated. Tasks and due rules with matching normalised names are deduplicated; non-matching items are appended to the target.

+
+
+ +
+ + + + + + + + + + + + +
Source dependencyCountMerge treatment
Firm selections{{ preview.counts.firm_selections }}Move or consolidate into target
System default tasks{{ preview.counts.system_default_tasks }}Move; matching task names deduplicated
Firm task templates{{ preview.counts.firm_task_templates }}Move; matching tenant/task names consolidated
Document requirements{{ preview.counts.document_requirements }}Remain with moved/consolidated task template
Document templates{{ preview.counts.document_templates }}Remain with moved/consolidated task template
Due-date rules{{ preview.counts.due_date_rules }}Move; matching rule names consolidated
Due-date extensions{{ preview.counts.due_date_extensions }}Move to target and surviving due rule
Client subscriptions{{ preview.counts.client_subscriptions_preserved }}Preserved on source for historical continuity
+
+ +
+ + + + + +
+ +
+
+ {% endif %} +
+{% endblock %} diff --git a/app/modules/services/ui.py b/app/modules/services/ui.py index f9ee3a3..686800c 100644 --- a/app/modules/services/ui.py +++ b/app/modules/services/ui.py @@ -1,7 +1,5 @@ from __future__ import annotations -from datetime import datetime, timezone - from fastapi import APIRouter, File, Form, Request, UploadFile from fastapi.responses import FileResponse, RedirectResponse, StreamingResponse from sqlalchemy import select @@ -32,7 +30,7 @@ from app.modules.services.due_dates import ( list_due_rules, parse_optional_date, ) -from app.modules.services.catalogue_export import build_service_catalogue_export +from app.modules.services.catalogue_merge import build_merge_preview, merge_service_catalogues from app.modules.services.task_documents import ( create_task_document_requirement, get_task_document_requirement, @@ -335,8 +333,8 @@ def catalogue_list(request: Request, q: str = '', category_id: int | None = None db.close() -@router.get('/catalogue/export') -def catalogue_export(request: Request): +@router.get('/catalogue/merge') +def catalogue_merge_page(request: Request, source_id: int | None = None, target_id: int | None = None): db = CommonSessionLocal() try: user = get_current_user(request, db=db) @@ -344,15 +342,88 @@ def catalogue_export(request: Request): return RedirectResponse(url='/login', status_code=303) if not _is_system_admin(db, user): return _redirect_denied() - require_permission(db, user, 'services.view') + require_permission(db, user, 'services.edit') - workbook_bytes = build_service_catalogue_export(db) - filename = f"service_catalogue_export_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.xlsx" - return StreamingResponse( - iter([workbook_bytes]), - media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - headers={'Content-Disposition': f'attachment; filename="{filename}"'}, + services = db.execute( + select(ServiceCatalogue).order_by(ServiceCatalogue.is_active.desc(), ServiceCatalogue.service_code.asc()) + ).scalars().all() + preview = None + error = None + if source_id and target_id: + try: + preview = build_merge_preview(db, source_id=source_id, target_id=target_id) + except ValueError as exc: + error = str(exc) + + return _render( + request, + 'modules/services/templates/services/catalogue_merge.html', + db, + user, + title='Merge Service Catalogue', + services=services, + source_id=source_id, + target_id=target_id, + preview=preview, + error=error, ) + except Exception: + return _redirect_denied() + finally: + db.close() + + +@router.post('/catalogue/merge') +def catalogue_merge_submit( + request: Request, + source_id: int = Form(...), + target_id: int = Form(...), + confirmation: str = Form(...), + csrf_token: str = Form(...), +): + validate_csrf(request, csrf_token) + db = CommonSessionLocal() + try: + user = get_current_user(request, db=db) + if not user: + return RedirectResponse(url='/login', status_code=303) + if not _is_system_admin(db, user): + return _redirect_denied() + require_permission(db, user, 'services.edit') + + preview = build_merge_preview(db, source_id=source_id, target_id=target_id) + expected = f"MERGE {preview.source.service_code} INTO {preview.target.service_code}" + if confirmation.strip().upper() != expected.upper(): + return RedirectResponse( + url=f'/services/catalogue/merge?source_id={source_id}&target_id={target_id}&error=confirmation', + status_code=303, + ) + + result = merge_service_catalogues( + db, + source_id=source_id, + target_id=target_id, + actor_user_id=user.id, + ) + db.commit() + + write_audit_log( + db, + action='services.catalogue.merged', + entity_type='service_catalogue', + entity_id=target_id, + entity_name=result['target_code'], + actor=user, + request=request, + details=result, + ) + return RedirectResponse(url=f'/services/catalogue/{target_id}?merged=1', status_code=303) + except ValueError: + db.rollback() + return RedirectResponse(url='/services/catalogue/merge?error=invalid', status_code=303) + except Exception: + db.rollback() + raise finally: db.close()