From 357305a687ee03cdea17c7a99c70b0d48ec316d7 Mon Sep 17 00:00:00 2001 From: Ali <010109ali@gmail.com> Date: Mon, 18 May 2026 12:51:55 +0000 Subject: [PATCH] refactor(excel-parser): extract header synonyms to its own module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The _HEADER_SYNONYMS dict, the Azərbaycan-to-Latin translation table, and the matcher helpers were sitting in the middle of excel_parser.py making the file harder to scan. Moved them to a dedicated header_synonyms.py with a docstring explaining how to extend it. Public API of the new module is just guess_standard_field(header) and HEADER_SYNONYMS — that's the surface excel_parser actually uses. --- jey_erp/bank_integration/excel_parser.py | 107 +---------------- jey_erp/bank_integration/header_synonyms.py | 124 ++++++++++++++++++++ 2 files changed, 127 insertions(+), 104 deletions(-) create mode 100644 jey_erp/bank_integration/header_synonyms.py diff --git a/jey_erp/bank_integration/excel_parser.py b/jey_erp/bank_integration/excel_parser.py index c7ec5ae..efdc1b6 100644 --- a/jey_erp/bank_integration/excel_parser.py +++ b/jey_erp/bank_integration/excel_parser.py @@ -382,109 +382,6 @@ def _parse_date(value, custom_format): return None -# Heuristic synonyms for auto-detection. Lowercased; the matcher normalises -# whitespace and Azərbaycan letters to Latin equivalents before comparing. -# Order within each list does not matter — first hit wins for the column. -# Order between standard fields matters: a header that could be both Purpose -# and Description matches Purpose first (Purpose comes earlier in the dict). -_HEADER_SYNONYMS = { - "Date": ( - "date", "operation date", "transaction date", "value date", "posting date", - "tarix", "tarıx", "əməliyyat tarixi", "tarix gun", - "дата", "дата операции", "дата документа", - ), - "Reference Number": ( - "reference", "reference number", "ref", "ref no", "ref number", - "doc no", "document no", "document number", "transaction id", "txn id", - "sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi", - "номер документа", "номер", "док", "ссылка", "идентификатор", - ), - "Counterparty": ( - "counterparty", "client name", "client", "customer name", "name", "partner", - "müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil", - "контрагент", "наименование", "плательщик", "получатель", - ), - "Counterparty Tax ID (VOEN)": ( - "voen", "tax id", "taxpayer id", "tin", "inn", "vat number", - "müştəri voen", "qarşı tərəf voen", - "инн", "налоговый номер", - ), - "Counterparty IBAN": ( - "iban", "counterparty iban", "client iban", "client account", - "müştəri iban", "müştəri hesab", "hesab nömrəsi", - "счёт", "счет", "номер счёта", "номер счета", - ), - "Amount": ( - "amount", "sum", "total", "value", - "məbləğ", - "сумма", - ), - "Debit": ( - "debit", "withdrawal", "pay", "outflow", "debet", - "məxaric", "ödəniş", - "дебет", "расход", "списание", - ), - "Credit": ( - "credit", "deposit", "receive", "inflow", "kredit", - "mədaxil", "daxilolma", - "кредит", "приход", "поступление", "зачисление", - ), - "Direction": ( - "direction", "type", "drcr", "operation type", "movement", - "məbləğ növü", "əməliyyat növü", "növ", - "тип", "тип операции", - ), - "Currency": ( - "currency", "ccy", "cur", - "valyuta", - "валюта", - ), - "Purpose": ( - "purpose", "narrative", "details", "purpose of payment", "payment purpose", - "təyinat", "ödənişin təyinatı", "ödəniş təyinatı", - "назначение", "назначение платежа", "основание", - ), - "Description": ( - "description", "comment", "comments", "remark", "remarks", "note", "notes", "memo", - "qeyd", "şərh", "izahat", - "комментарий", "примечание", "описание", - ), -} - - -_AZERI_TO_LATIN = str.maketrans({ - "ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o", - "ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c", - "ğ": "g", "Ğ": "g", -}) - - -def _norm_for_match(s): - if s is None: - return "" - s = str(s).strip().lower().translate(_AZERI_TO_LATIN) - return " ".join(s.split()) - - -def _guess_standard_field(header): - """Return a Standard Field label for the given header text, or None.""" - h = _norm_for_match(header) - if not h: - return None - # 1. Exact match first. - for field, synonyms in _HEADER_SYNONYMS.items(): - for syn in synonyms: - if h == _norm_for_match(syn): - return field - # 2. Substring fallback: header *contains* a synonym (e.g. "Payment Reference No."). - for field, synonyms in _HEADER_SYNONYMS.items(): - for syn in synonyms: - syn_n = _norm_for_match(syn) - if syn_n and len(syn_n) >= 4 and syn_n in h: - return field - return None - - @frappe.whitelist() def auto_detect_column_mappings(file_url, header_row=1): """Read sample file headers and propose a mapping to Standard Fields. @@ -503,6 +400,8 @@ def auto_detect_column_mappings(file_url, header_row=1): if not headers_result.get("success"): return headers_result + from jey_erp.bank_integration.header_synonyms import guess_standard_field + headers = headers_result.get("headers") or [] mappings = [] unmatched = [] @@ -511,7 +410,7 @@ def auto_detect_column_mappings(file_url, header_row=1): # the loop so both can be picked up.) used = set() for h in headers: - guess = _guess_standard_field(h) + guess = guess_standard_field(h) if guess and guess not in used: mappings.append({"excel_column": h, "standard_field": guess}) # Date / Reference / Amount / etc. should appear only once; Debit diff --git a/jey_erp/bank_integration/header_synonyms.py b/jey_erp/bank_integration/header_synonyms.py new file mode 100644 index 0000000..cde5e22 --- /dev/null +++ b/jey_erp/bank_integration/header_synonyms.py @@ -0,0 +1,124 @@ +"""Header-to-StandardField heuristic dictionary + matcher. + +Used by `excel_parser.auto_detect_column_mappings` to guess a column mapping +when the user uploads a sample bank statement. Kept in its own module so the +synonym lists can be edited (or extended per-locale) without touching the +parser. + +Adding a new synonym: append to the appropriate tuple in HEADER_SYNONYMS. +Adding a new language: just add more entries — strings are lower-cased and +Azərbaycan-specific letters are transliterated to Latin before matching, so +mixed-case / mixed-script entries are fine. +""" + + +# Synonyms are lowercased; the matcher normalises whitespace and Azərbaycan +# letters to Latin equivalents before comparing, so case and the ə/ş/ç/ğ +# letters don't matter here. +# +# Order WITHIN each tuple does not matter — first hit wins for the column. +# Order BETWEEN standard fields matters when two fields share a synonym +# (e.g. "description" appears in both Purpose and Description's natural- +# language pool): the dict order decides which wins. Purpose is listed +# before Description so a bank using "Purpose / Description" in one column +# lands on Purpose, which is the more useful classification downstream. +HEADER_SYNONYMS = { + "Date": ( + "date", "operation date", "transaction date", "value date", "posting date", + "tarix", "tarıx", "əməliyyat tarixi", "tarix gun", + "дата", "дата операции", "дата документа", + ), + "Reference Number": ( + "reference", "reference number", "ref", "ref no", "ref number", + "doc no", "document no", "document number", "transaction id", "txn id", + "sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi", + "номер документа", "номер", "док", "ссылка", "идентификатор", + ), + "Counterparty": ( + "counterparty", "client name", "client", "customer name", "name", "partner", + "müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil", + "контрагент", "наименование", "плательщик", "получатель", + ), + "Counterparty Tax ID (VOEN)": ( + "voen", "tax id", "taxpayer id", "tin", "inn", "vat number", + "müştəri voen", "qarşı tərəf voen", + "инн", "налоговый номер", + ), + "Counterparty IBAN": ( + "iban", "counterparty iban", "client iban", "client account", + "müştəri iban", "müştəri hesab", "hesab nömrəsi", + "счёт", "счет", "номер счёта", "номер счета", + ), + "Amount": ( + "amount", "sum", "total", "value", + "məbləğ", + "сумма", + ), + "Debit": ( + "debit", "withdrawal", "pay", "outflow", "debet", + "məxaric", "ödəniş", + "дебет", "расход", "списание", + ), + "Credit": ( + "credit", "deposit", "receive", "inflow", "kredit", + "mədaxil", "daxilolma", + "кредит", "приход", "поступление", "зачисление", + ), + "Direction": ( + "direction", "type", "drcr", "operation type", "movement", + "məbləğ növü", "əməliyyat növü", "növ", + "тип", "тип операции", + ), + "Currency": ( + "currency", "ccy", "cur", + "valyuta", + "валюта", + ), + "Purpose": ( + "purpose", "narrative", "details", "purpose of payment", "payment purpose", + "təyinat", "ödənişin təyinatı", "ödəniş təyinatı", + "назначение", "назначение платежа", "основание", + ), + "Description": ( + "description", "comment", "comments", "remark", "remarks", "note", "notes", "memo", + "qeyd", "şərh", "izahat", + "комментарий", "примечание", "описание", + ), +} + + +_AZERI_TO_LATIN = str.maketrans({ + "ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o", + "ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c", + "ğ": "g", "Ğ": "g", +}) + + +def _norm(s): + if s is None: + return "" + s = str(s).strip().lower().translate(_AZERI_TO_LATIN) + return " ".join(s.split()) + + +def guess_standard_field(header): + """Return a Standard Field label for the given header text, or None. + + Two-pass: exact normalised match first, then substring fallback (catches + headers like "Payment Reference No." against the synonym "reference"). The + substring pass requires a minimum length of 4 to avoid noise from very + short synonyms. + """ + h = _norm(header) + if not h: + return None + for field, synonyms in HEADER_SYNONYMS.items(): + for syn in synonyms: + if h == _norm(syn): + return field + for field, synonyms in HEADER_SYNONYMS.items(): + for syn in synonyms: + syn_n = _norm(syn) + if syn_n and len(syn_n) >= 4 and syn_n in h: + return field + return None