refactor(excel-parser): extract header synonyms to its own module
The _HEADER_SYNONYMS dict, the Azərbaycan-to-Latin translation table, and the matcher helpers were sitting in the middle of excel_parser.py making the file harder to scan. Moved them to a dedicated header_synonyms.py with a docstring explaining how to extend it. Public API of the new module is just guess_standard_field(header) and HEADER_SYNONYMS — that's the surface excel_parser actually uses.
This commit is contained in:
parent
381aebdbdb
commit
357305a687
|
|
@ -382,109 +382,6 @@ def _parse_date(value, custom_format):
|
|||
return None
|
||||
|
||||
|
||||
# Heuristic synonyms for auto-detection. Lowercased; the matcher normalises
|
||||
# whitespace and Azərbaycan letters to Latin equivalents before comparing.
|
||||
# Order within each list does not matter — first hit wins for the column.
|
||||
# Order between standard fields matters: a header that could be both Purpose
|
||||
# and Description matches Purpose first (Purpose comes earlier in the dict).
|
||||
_HEADER_SYNONYMS = {
|
||||
"Date": (
|
||||
"date", "operation date", "transaction date", "value date", "posting date",
|
||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||
"дата", "дата операции", "дата документа",
|
||||
),
|
||||
"Reference Number": (
|
||||
"reference", "reference number", "ref", "ref no", "ref number",
|
||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
||||
),
|
||||
"Counterparty": (
|
||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||
"контрагент", "наименование", "плательщик", "получатель",
|
||||
),
|
||||
"Counterparty Tax ID (VOEN)": (
|
||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||
"müştəri voen", "qarşı tərəf voen",
|
||||
"инн", "налоговый номер",
|
||||
),
|
||||
"Counterparty IBAN": (
|
||||
"iban", "counterparty iban", "client iban", "client account",
|
||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||
"счёт", "счет", "номер счёта", "номер счета",
|
||||
),
|
||||
"Amount": (
|
||||
"amount", "sum", "total", "value",
|
||||
"məbləğ",
|
||||
"сумма",
|
||||
),
|
||||
"Debit": (
|
||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||
"məxaric", "ödəniş",
|
||||
"дебет", "расход", "списание",
|
||||
),
|
||||
"Credit": (
|
||||
"credit", "deposit", "receive", "inflow", "kredit",
|
||||
"mədaxil", "daxilolma",
|
||||
"кредит", "приход", "поступление", "зачисление",
|
||||
),
|
||||
"Direction": (
|
||||
"direction", "type", "drcr", "operation type", "movement",
|
||||
"məbləğ növü", "əməliyyat növü", "növ",
|
||||
"тип", "тип операции",
|
||||
),
|
||||
"Currency": (
|
||||
"currency", "ccy", "cur",
|
||||
"valyuta",
|
||||
"валюта",
|
||||
),
|
||||
"Purpose": (
|
||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||
"назначение", "назначение платежа", "основание",
|
||||
),
|
||||
"Description": (
|
||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||
"qeyd", "şərh", "izahat",
|
||||
"комментарий", "примечание", "описание",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_AZERI_TO_LATIN = str.maketrans({
|
||||
"ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o",
|
||||
"ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c",
|
||||
"ğ": "g", "Ğ": "g",
|
||||
})
|
||||
|
||||
|
||||
def _norm_for_match(s):
|
||||
if s is None:
|
||||
return ""
|
||||
s = str(s).strip().lower().translate(_AZERI_TO_LATIN)
|
||||
return " ".join(s.split())
|
||||
|
||||
|
||||
def _guess_standard_field(header):
|
||||
"""Return a Standard Field label for the given header text, or None."""
|
||||
h = _norm_for_match(header)
|
||||
if not h:
|
||||
return None
|
||||
# 1. Exact match first.
|
||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
if h == _norm_for_match(syn):
|
||||
return field
|
||||
# 2. Substring fallback: header *contains* a synonym (e.g. "Payment Reference No.").
|
||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
syn_n = _norm_for_match(syn)
|
||||
if syn_n and len(syn_n) >= 4 and syn_n in h:
|
||||
return field
|
||||
return None
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def auto_detect_column_mappings(file_url, header_row=1):
|
||||
"""Read sample file headers and propose a mapping to Standard Fields.
|
||||
|
|
@ -503,6 +400,8 @@ def auto_detect_column_mappings(file_url, header_row=1):
|
|||
if not headers_result.get("success"):
|
||||
return headers_result
|
||||
|
||||
from jey_erp.bank_integration.header_synonyms import guess_standard_field
|
||||
|
||||
headers = headers_result.get("headers") or []
|
||||
mappings = []
|
||||
unmatched = []
|
||||
|
|
@ -511,7 +410,7 @@ def auto_detect_column_mappings(file_url, header_row=1):
|
|||
# the loop so both can be picked up.)
|
||||
used = set()
|
||||
for h in headers:
|
||||
guess = _guess_standard_field(h)
|
||||
guess = guess_standard_field(h)
|
||||
if guess and guess not in used:
|
||||
mappings.append({"excel_column": h, "standard_field": guess})
|
||||
# Date / Reference / Amount / etc. should appear only once; Debit
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
"""Header-to-StandardField heuristic dictionary + matcher.
|
||||
|
||||
Used by `excel_parser.auto_detect_column_mappings` to guess a column mapping
|
||||
when the user uploads a sample bank statement. Kept in its own module so the
|
||||
synonym lists can be edited (or extended per-locale) without touching the
|
||||
parser.
|
||||
|
||||
Adding a new synonym: append to the appropriate tuple in HEADER_SYNONYMS.
|
||||
Adding a new language: just add more entries — strings are lower-cased and
|
||||
Azərbaycan-specific letters are transliterated to Latin before matching, so
|
||||
mixed-case / mixed-script entries are fine.
|
||||
"""
|
||||
|
||||
|
||||
# Synonyms are lowercased; the matcher normalises whitespace and Azərbaycan
|
||||
# letters to Latin equivalents before comparing, so case and the ə/ş/ç/ğ
|
||||
# letters don't matter here.
|
||||
#
|
||||
# Order WITHIN each tuple does not matter — first hit wins for the column.
|
||||
# Order BETWEEN standard fields matters when two fields share a synonym
|
||||
# (e.g. "description" appears in both Purpose and Description's natural-
|
||||
# language pool): the dict order decides which wins. Purpose is listed
|
||||
# before Description so a bank using "Purpose / Description" in one column
|
||||
# lands on Purpose, which is the more useful classification downstream.
|
||||
HEADER_SYNONYMS = {
|
||||
"Date": (
|
||||
"date", "operation date", "transaction date", "value date", "posting date",
|
||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||
"дата", "дата операции", "дата документа",
|
||||
),
|
||||
"Reference Number": (
|
||||
"reference", "reference number", "ref", "ref no", "ref number",
|
||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
||||
),
|
||||
"Counterparty": (
|
||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||
"контрагент", "наименование", "плательщик", "получатель",
|
||||
),
|
||||
"Counterparty Tax ID (VOEN)": (
|
||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||
"müştəri voen", "qarşı tərəf voen",
|
||||
"инн", "налоговый номер",
|
||||
),
|
||||
"Counterparty IBAN": (
|
||||
"iban", "counterparty iban", "client iban", "client account",
|
||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||
"счёт", "счет", "номер счёта", "номер счета",
|
||||
),
|
||||
"Amount": (
|
||||
"amount", "sum", "total", "value",
|
||||
"məbləğ",
|
||||
"сумма",
|
||||
),
|
||||
"Debit": (
|
||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||
"məxaric", "ödəniş",
|
||||
"дебет", "расход", "списание",
|
||||
),
|
||||
"Credit": (
|
||||
"credit", "deposit", "receive", "inflow", "kredit",
|
||||
"mədaxil", "daxilolma",
|
||||
"кредит", "приход", "поступление", "зачисление",
|
||||
),
|
||||
"Direction": (
|
||||
"direction", "type", "drcr", "operation type", "movement",
|
||||
"məbləğ növü", "əməliyyat növü", "növ",
|
||||
"тип", "тип операции",
|
||||
),
|
||||
"Currency": (
|
||||
"currency", "ccy", "cur",
|
||||
"valyuta",
|
||||
"валюта",
|
||||
),
|
||||
"Purpose": (
|
||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||
"назначение", "назначение платежа", "основание",
|
||||
),
|
||||
"Description": (
|
||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||
"qeyd", "şərh", "izahat",
|
||||
"комментарий", "примечание", "описание",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_AZERI_TO_LATIN = str.maketrans({
|
||||
"ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o",
|
||||
"ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c",
|
||||
"ğ": "g", "Ğ": "g",
|
||||
})
|
||||
|
||||
|
||||
def _norm(s):
|
||||
if s is None:
|
||||
return ""
|
||||
s = str(s).strip().lower().translate(_AZERI_TO_LATIN)
|
||||
return " ".join(s.split())
|
||||
|
||||
|
||||
def guess_standard_field(header):
|
||||
"""Return a Standard Field label for the given header text, or None.
|
||||
|
||||
Two-pass: exact normalised match first, then substring fallback (catches
|
||||
headers like "Payment Reference No." against the synonym "reference"). The
|
||||
substring pass requires a minimum length of 4 to avoid noise from very
|
||||
short synonyms.
|
||||
"""
|
||||
h = _norm(header)
|
||||
if not h:
|
||||
return None
|
||||
for field, synonyms in HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
if h == _norm(syn):
|
||||
return field
|
||||
for field, synonyms in HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
syn_n = _norm(syn)
|
||||
if syn_n and len(syn_n) >= 4 and syn_n in h:
|
||||
return field
|
||||
return None
|
||||
Loading…
Reference in New Issue