Add admin task category import
This commit is contained in:
A R R R Associates
@@ -0,0 +1,307 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from fastapi import APIRouter, File, Form, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.db.common import CommonSessionLocal
|
||||
from app.core.security.csrf import get_or_create_csrf_token, validate_csrf
|
||||
from app.core.security.session_auth import get_current_user
|
||||
from app.core.templating import templates
|
||||
from app.modules.core.rbac.deps import get_user_roles
|
||||
from app.modules.services.models import (
|
||||
ClientServiceTaskInstance,
|
||||
FirmServiceTaskTemplate,
|
||||
ServiceCatalogue,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/services/task-categories", tags=["task-category-import-ui"])
|
||||
|
||||
ADMIN_ROLES = {"System Admin", "Firm Admin"}
|
||||
REQUIRED_COLUMNS = {"service_code", "task_name", "task_category"}
|
||||
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportRow:
|
||||
row_no: int
|
||||
service_code: str
|
||||
task_name: str
|
||||
task_category: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class RowResult:
|
||||
row_no: int
|
||||
service_code: str
|
||||
task_name: str
|
||||
task_category: str
|
||||
status: str
|
||||
message: str
|
||||
firm_template_id: int | None = None
|
||||
|
||||
|
||||
def _active_tenant_id(request: Request, user) -> int:
|
||||
value = (
|
||||
request.session.get("active_tenant_id")
|
||||
or request.session.get("selected_tenant_id")
|
||||
or request.session.get("tenant_id")
|
||||
or getattr(user, "tenant_id", None)
|
||||
)
|
||||
if not value:
|
||||
raise ValueError("No active tenant is selected.")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _normal(value) -> str:
|
||||
return " ".join(str(value or "").strip().split())
|
||||
|
||||
|
||||
def _key(value) -> str:
|
||||
return _normal(value).casefold()
|
||||
|
||||
|
||||
def _read_csv(data: bytes) -> list[dict[str, str]]:
|
||||
text = data.decode("utf-8-sig")
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
return [{str(k or "").strip(): "" if v is None else str(v) for k, v in row.items()} for row in reader]
|
||||
|
||||
|
||||
def _read_xlsx(data: bytes) -> list[dict[str, str]]:
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except ImportError as exc:
|
||||
raise ValueError("XLSX support requires openpyxl. Upload CSV instead or install openpyxl.") from exc
|
||||
wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
rows = ws.iter_rows(values_only=True)
|
||||
try:
|
||||
headers = [str(v or "").strip() for v in next(rows)]
|
||||
except StopIteration:
|
||||
return []
|
||||
result = []
|
||||
for values in rows:
|
||||
if all(v is None or str(v).strip() == "" for v in values):
|
||||
continue
|
||||
row = {}
|
||||
for i, header in enumerate(headers):
|
||||
if not header:
|
||||
continue
|
||||
value = values[i] if i < len(values) else None
|
||||
row[header] = "" if value is None else str(value)
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def _parse_upload(filename: str, data: bytes) -> list[ImportRow]:
|
||||
if not data:
|
||||
raise ValueError("The uploaded file is empty.")
|
||||
if len(data) > MAX_UPLOAD_BYTES:
|
||||
raise ValueError("The uploaded file exceeds the 5 MB limit.")
|
||||
lower = (filename or "").lower()
|
||||
if lower.endswith(".csv"):
|
||||
raw_rows = _read_csv(data)
|
||||
elif lower.endswith(".xlsx"):
|
||||
raw_rows = _read_xlsx(data)
|
||||
else:
|
||||
raise ValueError("Upload a .csv or .xlsx file.")
|
||||
if not raw_rows:
|
||||
raise ValueError("No data rows were found in the uploaded file.")
|
||||
|
||||
actual = {str(k).strip().casefold(): str(k).strip() for k in raw_rows[0].keys()}
|
||||
missing = [c for c in REQUIRED_COLUMNS if c.casefold() not in actual]
|
||||
if missing:
|
||||
raise ValueError("Missing required column(s): " + ", ".join(sorted(missing)))
|
||||
|
||||
def pick(row, name):
|
||||
for k, v in row.items():
|
||||
if str(k).strip().casefold() == name.casefold():
|
||||
return _normal(v)
|
||||
return ""
|
||||
|
||||
parsed: list[ImportRow] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for idx, row in enumerate(raw_rows, start=2):
|
||||
service_code = pick(row, "service_code")
|
||||
task_name = pick(row, "task_name")
|
||||
task_category = pick(row, "task_category")
|
||||
if not any((service_code, task_name, task_category)):
|
||||
continue
|
||||
if not service_code or not task_name or not task_category:
|
||||
raise ValueError(f"Row {idx}: service_code, task_name and task_category are all required.")
|
||||
pair = (_key(service_code), _key(task_name))
|
||||
if pair in seen:
|
||||
raise ValueError(f"Row {idx}: duplicate service/task mapping in the same upload.")
|
||||
seen.add(pair)
|
||||
parsed.append(ImportRow(idx, service_code, task_name, task_category))
|
||||
if not parsed:
|
||||
raise ValueError("No usable mappings were found in the uploaded file.")
|
||||
return parsed
|
||||
|
||||
|
||||
def _validate_rows(db, *, tenant_id: int, rows: Iterable[ImportRow]) -> list[RowResult]:
|
||||
results: list[RowResult] = []
|
||||
for row in rows:
|
||||
catalogues = db.execute(
|
||||
select(ServiceCatalogue).where(func.lower(ServiceCatalogue.service_code) == row.service_code.lower())
|
||||
).scalars().all()
|
||||
if len(catalogues) != 1:
|
||||
msg = "Service code not found." if not catalogues else "Service code is ambiguous."
|
||||
results.append(RowResult(row.row_no, row.service_code, row.task_name, row.task_category, "error", msg))
|
||||
continue
|
||||
catalogue = catalogues[0]
|
||||
templates = db.execute(
|
||||
select(FirmServiceTaskTemplate).where(
|
||||
FirmServiceTaskTemplate.tenant_id == tenant_id,
|
||||
FirmServiceTaskTemplate.service_catalogue_id == catalogue.id,
|
||||
FirmServiceTaskTemplate.is_active.is_(True),
|
||||
func.lower(FirmServiceTaskTemplate.task_name) == row.task_name.lower(),
|
||||
)
|
||||
).scalars().all()
|
||||
if len(templates) != 1:
|
||||
msg = "Active firm task template not found for this task name." if not templates else "Multiple active firm task templates have the same task name; no change made."
|
||||
results.append(RowResult(row.row_no, row.service_code, row.task_name, row.task_category, "error", msg))
|
||||
continue
|
||||
template = templates[0]
|
||||
current = _normal(getattr(template, "task_category", None))
|
||||
if current == row.task_category:
|
||||
status, msg = "unchanged", "Category already matches."
|
||||
else:
|
||||
status, msg = "ready", f"{current or '(blank)'} → {row.task_category}"
|
||||
results.append(RowResult(row.row_no, row.service_code, row.task_name, row.task_category, status, msg, int(template.id)))
|
||||
return results
|
||||
|
||||
|
||||
def _ctx(request, db, user, **extra):
|
||||
ctx = {
|
||||
"request": request,
|
||||
"current_user": user,
|
||||
"current_user_roles": get_user_roles(db, user.id),
|
||||
"csrf_token": get_or_create_csrf_token(request),
|
||||
"title": "Task Categories Import",
|
||||
}
|
||||
ctx.update(extra)
|
||||
return ctx
|
||||
|
||||
|
||||
def _admin_or_response(request: Request, db):
|
||||
user = get_current_user(request, db=db)
|
||||
if not user:
|
||||
return None, RedirectResponse(url="/login", status_code=303)
|
||||
roles = set(get_user_roles(db, user.id))
|
||||
if not roles.intersection(ADMIN_ROLES):
|
||||
try:
|
||||
from app.core.http_responses import ui_access_denied
|
||||
return None, ui_access_denied()
|
||||
except Exception:
|
||||
return None, Response("Access denied", status_code=403)
|
||||
return user, None
|
||||
|
||||
|
||||
@router.get("/import", response_class=HTMLResponse)
|
||||
def task_category_import_page(request: Request):
|
||||
db = CommonSessionLocal()
|
||||
try:
|
||||
user, response = _admin_or_response(request, db)
|
||||
if response:
|
||||
return response
|
||||
tenant_id = _active_tenant_id(request, user)
|
||||
return templates.TemplateResponse(
|
||||
"modules/services/templates/services/task_category_import.html",
|
||||
_ctx(request, db, user, tenant_id=tenant_id, results=[], errors=[], imported=False, summary=None),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.get("/template.csv")
|
||||
def task_category_import_template(request: Request):
|
||||
db = CommonSessionLocal()
|
||||
try:
|
||||
user, response = _admin_or_response(request, db)
|
||||
if response:
|
||||
return response
|
||||
content = (
|
||||
"service_code,task_name,task_category\r\n"
|
||||
"TAX-AUDIT-REPORT,Verify opening balances,Planning & Opening Checks\r\n"
|
||||
"TAX-AUDIT-REPORT,Verify fixed assets and depreciation,Fixed Assets & Depreciation\r\n"
|
||||
)
|
||||
return Response(
|
||||
content,
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": 'attachment; filename="task_category_import_template.csv"'},
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.post("/import", response_class=HTMLResponse)
|
||||
async def task_category_import_submit(
|
||||
request: Request,
|
||||
csrf_token: str = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
action: str = Form("validate"),
|
||||
update_existing_tasks: str | None = Form(None),
|
||||
):
|
||||
validate_csrf(request, csrf_token)
|
||||
db = CommonSessionLocal()
|
||||
try:
|
||||
user, response = _admin_or_response(request, db)
|
||||
if response:
|
||||
return response
|
||||
tenant_id = _active_tenant_id(request, user)
|
||||
errors: list[str] = []
|
||||
results: list[RowResult] = []
|
||||
imported = False
|
||||
summary = None
|
||||
try:
|
||||
data = await file.read()
|
||||
parsed = _parse_upload(file.filename or "", data)
|
||||
results = _validate_rows(db, tenant_id=tenant_id, rows=parsed)
|
||||
error_count = sum(1 for r in results if r.status == "error")
|
||||
ready_count = sum(1 for r in results if r.status == "ready")
|
||||
unchanged_count = sum(1 for r in results if r.status == "unchanged")
|
||||
summary = {"total": len(results), "ready": ready_count, "unchanged": unchanged_count, "errors": error_count, "existing_tasks_updated": 0}
|
||||
|
||||
if action == "import":
|
||||
if error_count:
|
||||
raise ValueError("Import cancelled because validation errors exist. Correct the file and upload it again; no database changes were made.")
|
||||
by_template = {r.firm_template_id: r for r in results if r.firm_template_id and r.status == "ready"}
|
||||
for template_id, result in by_template.items():
|
||||
template = db.get(FirmServiceTaskTemplate, template_id)
|
||||
if template is None or int(template.tenant_id) != tenant_id:
|
||||
raise ValueError(f"Row {result.row_no}: task template changed during import. Please retry.")
|
||||
template.task_category = result.task_category
|
||||
|
||||
if update_existing_tasks:
|
||||
existing = db.execute(
|
||||
select(ClientServiceTaskInstance).where(
|
||||
ClientServiceTaskInstance.tenant_id == tenant_id,
|
||||
ClientServiceTaskInstance.firm_task_template_id == template.id,
|
||||
ClientServiceTaskInstance.is_active.is_(True),
|
||||
)
|
||||
).scalars().all()
|
||||
for task in existing:
|
||||
task.task_category = result.task_category
|
||||
summary["existing_tasks_updated"] += len(existing)
|
||||
db.commit()
|
||||
imported = True
|
||||
for r in results:
|
||||
if r.status == "ready":
|
||||
r.status = "updated"
|
||||
r.message = "Category updated successfully."
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
errors.append(str(exc))
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"modules/services/templates/services/task_category_import.html",
|
||||
_ctx(request, db, user, tenant_id=tenant_id, results=results, errors=errors, imported=imported, summary=summary),
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,76 @@
|
||||
{% extends "base/layout.html" %}
|
||||
{% block content %}
|
||||
<div class="max-w-6xl mx-auto px-4 py-6 space-y-6">
|
||||
<div class="flex flex-wrap items-start justify-between gap-3">
|
||||
<div>
|
||||
<h1 class="text-2xl font-semibold text-slate-900">Task Categories Import</h1>
|
||||
<p class="mt-1 text-sm text-slate-600">Assign task categories to existing firm task templates for the active tenant. Validation is performed before any update.</p>
|
||||
</div>
|
||||
<a href="/services/task-categories/template.csv" class="inline-flex items-center rounded-lg border border-slate-300 bg-white px-4 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50">Download CSV template</a>
|
||||
</div>
|
||||
|
||||
<div class="rounded-xl border border-amber-200 bg-amber-50 p-4 text-sm text-amber-900">
|
||||
<strong>Active tenant:</strong> {{ tenant_id }}. Required columns are <code>service_code</code>, <code>task_name</code> and <code>task_category</code>. Matching is intentionally strict so an incorrect category is never applied to the wrong task.
|
||||
</div>
|
||||
|
||||
{% if errors %}
|
||||
<div class="rounded-xl border border-red-200 bg-red-50 p-4">
|
||||
<div class="font-semibold text-red-800">Import not applied</div>
|
||||
<ul class="mt-2 list-disc pl-5 text-sm text-red-700">
|
||||
{% for error in errors %}<li>{{ error }}</li>{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if imported %}
|
||||
<div class="rounded-xl border border-emerald-200 bg-emerald-50 p-4 text-sm text-emerald-800">
|
||||
Task categories were updated successfully.
|
||||
{% if summary and summary.existing_tasks_updated %} {{ summary.existing_tasks_updated }} existing generated task(s) were also synchronized.{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" enctype="multipart/form-data" class="rounded-xl border border-slate-200 bg-white p-5 shadow-sm space-y-4">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-slate-700">CSV or XLSX file</label>
|
||||
<input type="file" name="file" accept=".csv,.xlsx" required class="mt-2 block w-full rounded-lg border border-slate-300 px-3 py-2 text-sm">
|
||||
</div>
|
||||
<label class="flex items-start gap-3 text-sm text-slate-700">
|
||||
<input type="checkbox" name="update_existing_tasks" value="1" class="mt-1 rounded border-slate-300">
|
||||
<span><strong>Also update existing generated tasks.</strong><br><span class="text-slate-500">Leave this unticked if you only want future/generated tasks to use the new categories. Existing completed/history tasks are otherwise left untouched.</span></span>
|
||||
</label>
|
||||
<div class="flex flex-wrap gap-3">
|
||||
<button type="submit" name="action" value="validate" class="rounded-lg border border-slate-300 bg-white px-4 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50">Validate only</button>
|
||||
<button type="submit" name="action" value="import" class="rounded-lg bg-slate-900 px-4 py-2 text-sm font-medium text-white hover:bg-slate-800">Import now</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{% if summary %}
|
||||
<div class="grid grid-cols-2 gap-3 md:grid-cols-4">
|
||||
<div class="rounded-xl border bg-white p-4"><div class="text-xs uppercase text-slate-500">Rows</div><div class="text-xl font-semibold">{{ summary.total }}</div></div>
|
||||
<div class="rounded-xl border bg-white p-4"><div class="text-xs uppercase text-slate-500">Ready</div><div class="text-xl font-semibold">{{ summary.ready }}</div></div>
|
||||
<div class="rounded-xl border bg-white p-4"><div class="text-xs uppercase text-slate-500">Unchanged</div><div class="text-xl font-semibold">{{ summary.unchanged }}</div></div>
|
||||
<div class="rounded-xl border bg-white p-4"><div class="text-xs uppercase text-slate-500">Errors</div><div class="text-xl font-semibold">{{ summary.errors }}</div></div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if results %}
|
||||
<div class="overflow-x-auto rounded-xl border border-slate-200 bg-white shadow-sm">
|
||||
<table class="min-w-full divide-y divide-slate-200 text-sm">
|
||||
<thead class="bg-slate-50 text-left text-xs font-semibold uppercase tracking-wide text-slate-500">
|
||||
<tr><th class="px-4 py-3">Row</th><th class="px-4 py-3">Service</th><th class="px-4 py-3">Task</th><th class="px-4 py-3">Category</th><th class="px-4 py-3">Status</th><th class="px-4 py-3">Result</th></tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-slate-100">
|
||||
{% for row in results %}
|
||||
<tr>
|
||||
<td class="px-4 py-3">{{ row.row_no }}</td><td class="px-4 py-3 font-medium">{{ row.service_code }}</td><td class="px-4 py-3">{{ row.task_name }}</td><td class="px-4 py-3">{{ row.task_category }}</td>
|
||||
<td class="px-4 py-3"><span class="rounded-full px-2 py-1 text-xs {% if row.status == 'error' %}bg-red-100 text-red-700{% elif row.status == 'updated' %}bg-emerald-100 text-emerald-700{% elif row.status == 'ready' %}bg-blue-100 text-blue-700{% else %}bg-slate-100 text-slate-700{% endif %}">{{ row.status }}</span></td>
|
||||
<td class="px-4 py-3 text-slate-600">{{ row.message }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -18,6 +18,7 @@ from app.modules.services.ui import router as services_ui_router
|
||||
from app.modules.services.engagements_ui import router as engagements_ui_router
|
||||
from app.modules.services.subscriptions_ui import router as service_subscriptions_ui_router
|
||||
from app.modules.services.work_tracker_ui import router as work_tracker_ui_router
|
||||
from app.modules.services.task_category_import import router as task_category_import_router
|
||||
from app.modules.billing.ui import router as billing_ui_router
|
||||
from app.modules.platform_billing.ui import router as platform_billing_ui_router
|
||||
from app.modules.marketplace.ui import router as marketplace_ui_router, public_router as marketplace_public_router
|
||||
@@ -128,6 +129,7 @@ def mount_ui(app: FastAPI) -> None:
|
||||
app.include_router(consultants_ui_router)
|
||||
app.include_router(engagements_ui_router)
|
||||
app.include_router(service_subscriptions_ui_router)
|
||||
app.include_router(task_category_import_router)
|
||||
app.include_router(client_portal_router)
|
||||
app.include_router(consultant_portal_router)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user