Add Phase 14 sales and income foundation
This commit is contained in:
A R R R Associates
@@ -0,0 +1,424 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.modules.accounting.sales_models import AccountingSalesIncomeTransaction
|
||||
from app.modules.accounting.sales_taxonomy import ensure_sales_income_taxonomy, sales_income_natures
|
||||
from app.modules.accounting.taxonomy_models import AccountingNature
|
||||
|
||||
|
||||
COLUMN_ALIASES = {
|
||||
"invoice_number": [
|
||||
"invoice number", "invoice no", "invoice no.", "inv no", "document number",
|
||||
"document no", "voucher number", "bill no", "bill number",
|
||||
],
|
||||
"invoice_date": ["invoice date", "document date", "date", "voucher date", "bill date"],
|
||||
"customer_name": [
|
||||
"customer name", "recipient name", "party name", "buyer name", "customer",
|
||||
"recipient", "party",
|
||||
],
|
||||
"customer_gstin": [
|
||||
"customer gstin", "recipient gstin", "gstin", "gstin/uin", "gstin uin",
|
||||
"buyer gstin",
|
||||
],
|
||||
"document_type": ["document type", "doc type", "invoice type", "voucher type"],
|
||||
"return_period": ["return period", "tax period", "period", "month"],
|
||||
"place_of_supply": ["place of supply", "pos", "state"],
|
||||
"supply_type": ["supply type", "nature of supply", "transaction type"],
|
||||
"export_type": ["export type", "exp type"],
|
||||
"hsn_code": ["hsn", "hsn code", "hsn/sac", "sac", "sac code"],
|
||||
"description_text": [
|
||||
"description", "item description", "product description", "particulars",
|
||||
"narration", "item name", "service description",
|
||||
],
|
||||
"taxable_value": ["taxable value", "taxable amount", "assessable value", "basic amount"],
|
||||
"cgst": ["cgst", "cgst amount"],
|
||||
"sgst": ["sgst", "sgst amount", "utgst", "utgst amount"],
|
||||
"igst": ["igst", "igst amount"],
|
||||
"cess": ["cess", "cess amount"],
|
||||
"invoice_value": [
|
||||
"invoice value", "invoice amount", "total invoice value", "total amount",
|
||||
"gross amount", "bill amount", "voucher amount",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _utcnow():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _key(value: Any) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", " ", str(value or "").strip().lower()).strip()
|
||||
|
||||
|
||||
def _s(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
try:
|
||||
if pd.isna(value):
|
||||
return ""
|
||||
except Exception:
|
||||
pass
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def _f(value: Any) -> float:
|
||||
if value is None:
|
||||
return 0.0
|
||||
try:
|
||||
if pd.isna(value):
|
||||
return 0.0
|
||||
except Exception:
|
||||
pass
|
||||
text = str(value).replace(",", "").replace("₹", "").strip()
|
||||
if text.startswith("(") and text.endswith(")"):
|
||||
text = "-" + text[1:-1]
|
||||
try:
|
||||
return round(float(text or 0), 2)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _date(value: Any) -> str:
|
||||
dt = pd.to_datetime(value, errors="coerce", dayfirst=True)
|
||||
if pd.isna(dt):
|
||||
return _s(value)[:20]
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _columns(frame: pd.DataFrame) -> dict[str, str]:
|
||||
keyed = {_key(column): column for column in frame.columns}
|
||||
result = {}
|
||||
for target, aliases in COLUMN_ALIASES.items():
|
||||
for alias in aliases:
|
||||
if _key(alias) in keyed:
|
||||
result[target] = keyed[_key(alias)]
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def _read_upload(filename: str, payload: bytes) -> pd.DataFrame:
|
||||
suffix = Path(filename or "").suffix.lower()
|
||||
if suffix == ".csv":
|
||||
return pd.read_csv(io.BytesIO(payload))
|
||||
if suffix in {".xlsx", ".xlsm", ".xls"}:
|
||||
book = pd.ExcelFile(io.BytesIO(payload))
|
||||
candidates = []
|
||||
for sheet in book.sheet_names:
|
||||
frame = pd.read_excel(book, sheet_name=sheet)
|
||||
mapping = _columns(frame)
|
||||
score = len(mapping)
|
||||
if "invoice_number" in mapping:
|
||||
score += 3
|
||||
if "customer_name" in mapping or "customer_gstin" in mapping:
|
||||
score += 2
|
||||
if "taxable_value" in mapping or "invoice_value" in mapping:
|
||||
score += 2
|
||||
candidates.append((score, sheet, frame))
|
||||
candidates.sort(key=lambda item: (-item[0], item[1]))
|
||||
if not candidates or candidates[0][0] < 3:
|
||||
raise ValueError(
|
||||
"No usable outward-sales sheet was found. Expected invoice/customer/value columns."
|
||||
)
|
||||
return candidates[0][2]
|
||||
raise ValueError("Sales & Income import supports CSV, XLSX, XLSM and XLS files.")
|
||||
|
||||
|
||||
def _nature_map(db, tenant_id: int):
|
||||
ensure_sales_income_taxonomy(db, tenant_id)
|
||||
rows = sales_income_natures(db, tenant_id)
|
||||
return {row.code: row for row in rows}
|
||||
|
||||
|
||||
def _classify(data: dict[str, Any], nature_by_code: dict[str, AccountingNature]):
|
||||
haystack = " ".join([
|
||||
_s(data.get("document_type")),
|
||||
_s(data.get("supply_type")),
|
||||
_s(data.get("export_type")),
|
||||
_s(data.get("description_text")),
|
||||
_s(data.get("customer_name")),
|
||||
]).upper()
|
||||
hsn = re.sub(r"\D", "", _s(data.get("hsn_code")))
|
||||
|
||||
def result(code, confidence, reason):
|
||||
nature = nature_by_code.get(code) or nature_by_code["SALES_REVIEW_REQUIRED"]
|
||||
return nature, confidence, reason
|
||||
|
||||
if any(token in haystack for token in ("EXPORT", "SEZ", "WITH PAYMENT", "WITHOUT PAYMENT")):
|
||||
return result("EXPORT_SALES", 92, "Export/SEZ wording in outward transaction.")
|
||||
if any(token in haystack for token in ("EXEMPT", "NIL RATED", "NIL-RATED", "NON GST", "NON-GST")):
|
||||
return result("EXEMPT_NIL_SALES", 92, "Exempt/nil/non-GST supply wording.")
|
||||
if any(token in haystack for token in ("SCRAP", "WASTE", "OLD NEWSPAPER", "METAL WASTE")):
|
||||
return result("SCRAP_SALES", 90, "Scrap/waste sale wording.")
|
||||
if any(token in haystack for token in ("COMMISSION", "BROKERAGE")):
|
||||
return result("COMMISSION_INCOME", 88, "Commission/brokerage income wording.")
|
||||
if any(token in haystack for token in ("RENTAL", "RENT ", "HIRE CHARGES", "LEASE INCOME")):
|
||||
return result("RENTAL_INCOME", 86, "Rental/hire/lease income wording.")
|
||||
if any(token in haystack for token in ("INTEREST", "FD INTEREST", "LOAN INTEREST")):
|
||||
return result("INTEREST_INCOME", 90, "Interest income wording.")
|
||||
if any(token in haystack for token in ("JOB WORK", "JOBWORK", "PROCESSING CHARGES", "CONVERSION CHARGES")):
|
||||
return result("JOB_WORK_INCOME", 88, "Job-work/processing income wording.")
|
||||
if any(token in haystack for token in (
|
||||
"PROFESSIONAL", "CONSULTANCY", "SERVICE CHARGES", "SERVICE FEE",
|
||||
"AUDIT FEE", "ACCOUNTING FEE", "MANAGEMENT FEE", "LABOUR CHARGES",
|
||||
)):
|
||||
return result("SERVICE_INCOME", 86, "Service/professional income wording.")
|
||||
if any(token in haystack for token in (
|
||||
"SALE OF VEHICLE", "SALE OF MACHINERY", "SALE OF ASSET",
|
||||
"OLD VEHICLE SALE", "OLD MACHINERY SALE",
|
||||
)):
|
||||
return result("ASSET_SALE", 90, "Capital-asset disposal wording.")
|
||||
|
||||
# SAC/service HSN chapters commonly begin 99. This is only a first-pass signal.
|
||||
if hsn.startswith("99"):
|
||||
return result("SERVICE_INCOME", 78, "SAC/HSN 99xx indicates service supply.")
|
||||
|
||||
# Generic outward invoice with goods HSN is treated conservatively as trading sales.
|
||||
if hsn and not hsn.startswith("99"):
|
||||
return result(
|
||||
"TRADING_SALES",
|
||||
68,
|
||||
"Goods HSN present; trading/manufacturing distinction requires business context or review.",
|
||||
)
|
||||
|
||||
if _s(data.get("description_text")):
|
||||
return result(
|
||||
"OTHER_OPERATING_INCOME",
|
||||
55,
|
||||
"Outward transaction has description but insufficient evidence for a narrower nature.",
|
||||
)
|
||||
return result("SALES_REVIEW_REQUIRED", 30, "Insufficient evidence for reliable income classification.")
|
||||
|
||||
|
||||
def _fingerprint(client_id: int, data: dict[str, Any]) -> str:
|
||||
raw = "|".join([
|
||||
str(client_id),
|
||||
_s(data.get("invoice_number")).upper(),
|
||||
_s(data.get("invoice_date")),
|
||||
_s(data.get("customer_gstin")).upper(),
|
||||
_s(data.get("customer_name")).upper(),
|
||||
f"{_f(data.get('taxable_value')):.2f}",
|
||||
f"{_f(data.get('invoice_value')):.2f}",
|
||||
_s(data.get("document_type")).upper(),
|
||||
])
|
||||
return hashlib.sha256(raw.encode("utf-8", "ignore")).hexdigest()
|
||||
|
||||
|
||||
def import_sales_file(
|
||||
db,
|
||||
*,
|
||||
tenant_id: int,
|
||||
client_id: int,
|
||||
filename: str,
|
||||
payload: bytes,
|
||||
user_id: int,
|
||||
source_type: str = "outward_import",
|
||||
):
|
||||
frame = _read_upload(filename, payload)
|
||||
mapping = _columns(frame)
|
||||
|
||||
required = {"invoice_number"}
|
||||
if not required.issubset(mapping):
|
||||
raise ValueError("Could not identify an Invoice Number column in the uploaded file.")
|
||||
if not ({"customer_name", "customer_gstin"} & set(mapping)):
|
||||
raise ValueError("Could not identify Customer Name or Customer GSTIN.")
|
||||
if not ({"taxable_value", "invoice_value"} & set(mapping)):
|
||||
raise ValueError("Could not identify Taxable Value or Invoice Value.")
|
||||
|
||||
nature_by_code = _nature_map(db, tenant_id)
|
||||
inserted = 0
|
||||
duplicates = 0
|
||||
skipped_blank = 0
|
||||
|
||||
for _, row in frame.iterrows():
|
||||
data = {
|
||||
target: row[column]
|
||||
for target, column in mapping.items()
|
||||
}
|
||||
data = {
|
||||
"invoice_number": _s(data.get("invoice_number")),
|
||||
"invoice_date": _date(data.get("invoice_date")),
|
||||
"customer_name": _s(data.get("customer_name")),
|
||||
"customer_gstin": re.sub(r"\s+", "", _s(data.get("customer_gstin")).upper()),
|
||||
"document_type": _s(data.get("document_type")) or "Invoice",
|
||||
"return_period": _s(data.get("return_period")),
|
||||
"place_of_supply": _s(data.get("place_of_supply")),
|
||||
"supply_type": _s(data.get("supply_type")),
|
||||
"export_type": _s(data.get("export_type")),
|
||||
"hsn_code": re.sub(r"\.0$", "", _s(data.get("hsn_code"))),
|
||||
"description_text": _s(data.get("description_text")),
|
||||
"taxable_value": _f(data.get("taxable_value")),
|
||||
"cgst": _f(data.get("cgst")),
|
||||
"sgst": _f(data.get("sgst")),
|
||||
"igst": _f(data.get("igst")),
|
||||
"cess": _f(data.get("cess")),
|
||||
"invoice_value": _f(data.get("invoice_value")),
|
||||
}
|
||||
|
||||
if not data["invoice_number"] and not data["customer_name"] and not data["customer_gstin"]:
|
||||
skipped_blank += 1
|
||||
continue
|
||||
if data["invoice_value"] == 0:
|
||||
data["invoice_value"] = round(
|
||||
data["taxable_value"] + data["cgst"] + data["sgst"] + data["igst"] + data["cess"],
|
||||
2,
|
||||
)
|
||||
|
||||
fp = _fingerprint(client_id, data)
|
||||
exists = db.execute(
|
||||
select(AccountingSalesIncomeTransaction.id).where(
|
||||
AccountingSalesIncomeTransaction.tenant_id == int(tenant_id),
|
||||
AccountingSalesIncomeTransaction.client_id == int(client_id),
|
||||
AccountingSalesIncomeTransaction.fingerprint == fp,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists:
|
||||
duplicates += 1
|
||||
continue
|
||||
|
||||
nature, confidence, reason = _classify(data, nature_by_code)
|
||||
record = AccountingSalesIncomeTransaction(
|
||||
tenant_id=int(tenant_id),
|
||||
client_id=int(client_id),
|
||||
source_type=source_type,
|
||||
source_file_name=Path(filename or "").name[:255],
|
||||
fingerprint=fp,
|
||||
return_period=data["return_period"],
|
||||
document_type=data["document_type"],
|
||||
invoice_number=data["invoice_number"],
|
||||
invoice_date=data["invoice_date"],
|
||||
customer_name=data["customer_name"],
|
||||
customer_gstin=data["customer_gstin"],
|
||||
place_of_supply=data["place_of_supply"],
|
||||
supply_type=data["supply_type"],
|
||||
export_type=data["export_type"],
|
||||
hsn_code=data["hsn_code"],
|
||||
description_text=data["description_text"],
|
||||
taxable_value=data["taxable_value"],
|
||||
cgst=data["cgst"],
|
||||
sgst=data["sgst"],
|
||||
igst=data["igst"],
|
||||
cess=data["cess"],
|
||||
invoice_value=data["invoice_value"],
|
||||
suggested_nature_id=nature.id,
|
||||
suggested_confidence=int(confidence),
|
||||
suggestion_reason=reason,
|
||||
review_status="suggested" if confidence >= 60 else "review_required",
|
||||
created_by_user_id=user_id,
|
||||
)
|
||||
db.add(record)
|
||||
inserted += 1
|
||||
|
||||
db.commit()
|
||||
return {
|
||||
"inserted": inserted,
|
||||
"duplicates": duplicates,
|
||||
"skipped_blank": skipped_blank,
|
||||
"source_rows": len(frame),
|
||||
}
|
||||
|
||||
|
||||
def list_sales(
|
||||
db,
|
||||
*,
|
||||
tenant_id: int,
|
||||
client_id: int,
|
||||
status: str = "",
|
||||
customer: str = "",
|
||||
period: str = "",
|
||||
page: int = 1,
|
||||
per_page: int = 25,
|
||||
):
|
||||
where = [
|
||||
AccountingSalesIncomeTransaction.tenant_id == int(tenant_id),
|
||||
AccountingSalesIncomeTransaction.client_id == int(client_id),
|
||||
]
|
||||
if status and status != "all":
|
||||
where.append(AccountingSalesIncomeTransaction.review_status == status)
|
||||
if customer.strip():
|
||||
term = f"%{customer.strip()}%"
|
||||
where.append(
|
||||
(
|
||||
AccountingSalesIncomeTransaction.customer_name.ilike(term)
|
||||
| AccountingSalesIncomeTransaction.customer_gstin.ilike(term)
|
||||
)
|
||||
)
|
||||
if period.strip():
|
||||
where.append(AccountingSalesIncomeTransaction.return_period == period.strip())
|
||||
|
||||
total = int(
|
||||
db.scalar(
|
||||
select(func.count(AccountingSalesIncomeTransaction.id)).where(*where)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
per_page = max(10, min(100, int(per_page or 25)))
|
||||
pages = max(1, (total + per_page - 1) // per_page)
|
||||
page = max(1, min(int(page or 1), pages))
|
||||
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(AccountingSalesIncomeTransaction)
|
||||
.where(*where)
|
||||
.order_by(
|
||||
AccountingSalesIncomeTransaction.invoice_date.desc(),
|
||||
AccountingSalesIncomeTransaction.id.desc(),
|
||||
)
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
).scalars().all()
|
||||
)
|
||||
return rows, total, page, pages
|
||||
|
||||
|
||||
def review_sales(
|
||||
db,
|
||||
*,
|
||||
tenant_id: int,
|
||||
client_id: int,
|
||||
row_id: int,
|
||||
final_nature_id: int,
|
||||
final_ledger_name: str,
|
||||
review_note: str,
|
||||
user_id: int,
|
||||
):
|
||||
row = db.execute(
|
||||
select(AccountingSalesIncomeTransaction).where(
|
||||
AccountingSalesIncomeTransaction.id == int(row_id),
|
||||
AccountingSalesIncomeTransaction.tenant_id == int(tenant_id),
|
||||
AccountingSalesIncomeTransaction.client_id == int(client_id),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not row:
|
||||
raise ValueError("Sales / income transaction was not found.")
|
||||
|
||||
nature = db.execute(
|
||||
select(AccountingNature).where(
|
||||
AccountingNature.id == int(final_nature_id),
|
||||
AccountingNature.tenant_id == int(tenant_id),
|
||||
AccountingNature.classification_group == "income",
|
||||
AccountingNature.is_posting_nature.is_(True),
|
||||
AccountingNature.is_active.is_(True),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not nature:
|
||||
raise ValueError("Select an active Sales & Income accounting nature.")
|
||||
|
||||
row.final_nature_id = nature.id
|
||||
row.final_ledger_name = (final_ledger_name or "").strip()
|
||||
row.review_note = (review_note or "").strip() or None
|
||||
row.review_status = "reviewed"
|
||||
row.reviewed_by_user_id = user_id
|
||||
row.reviewed_at_utc = _utcnow()
|
||||
db.add(row)
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return row
|
||||
Reference in New Issue
Block a user