refactor(excel-parser): extract header synonyms to its own module
The _HEADER_SYNONYMS dict, the Azərbaycan-to-Latin translation table, and the matcher helpers were sitting in the middle of excel_parser.py making the file harder to scan. Moved them to a dedicated header_synonyms.py with a docstring explaining how to extend it. Public API of the new module is just guess_standard_field(header) and HEADER_SYNONYMS — that's the surface excel_parser actually uses.
This commit is contained in:
parent
381aebdbdb
commit
357305a687
|
|
@ -382,109 +382,6 @@ def _parse_date(value, custom_format):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
# Heuristic synonyms for auto-detection. Lowercased; the matcher normalises
|
|
||||||
# whitespace and Azərbaycan letters to Latin equivalents before comparing.
|
|
||||||
# Order within each list does not matter — first hit wins for the column.
|
|
||||||
# Order between standard fields matters: a header that could be both Purpose
|
|
||||||
# and Description matches Purpose first (Purpose comes earlier in the dict).
|
|
||||||
_HEADER_SYNONYMS = {
|
|
||||||
"Date": (
|
|
||||||
"date", "operation date", "transaction date", "value date", "posting date",
|
|
||||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
|
||||||
"дата", "дата операции", "дата документа",
|
|
||||||
),
|
|
||||||
"Reference Number": (
|
|
||||||
"reference", "reference number", "ref", "ref no", "ref number",
|
|
||||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
|
||||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
|
||||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
|
||||||
),
|
|
||||||
"Counterparty": (
|
|
||||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
|
||||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
|
||||||
"контрагент", "наименование", "плательщик", "получатель",
|
|
||||||
),
|
|
||||||
"Counterparty Tax ID (VOEN)": (
|
|
||||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
|
||||||
"müştəri voen", "qarşı tərəf voen",
|
|
||||||
"инн", "налоговый номер",
|
|
||||||
),
|
|
||||||
"Counterparty IBAN": (
|
|
||||||
"iban", "counterparty iban", "client iban", "client account",
|
|
||||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
|
||||||
"счёт", "счет", "номер счёта", "номер счета",
|
|
||||||
),
|
|
||||||
"Amount": (
|
|
||||||
"amount", "sum", "total", "value",
|
|
||||||
"məbləğ",
|
|
||||||
"сумма",
|
|
||||||
),
|
|
||||||
"Debit": (
|
|
||||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
|
||||||
"məxaric", "ödəniş",
|
|
||||||
"дебет", "расход", "списание",
|
|
||||||
),
|
|
||||||
"Credit": (
|
|
||||||
"credit", "deposit", "receive", "inflow", "kredit",
|
|
||||||
"mədaxil", "daxilolma",
|
|
||||||
"кредит", "приход", "поступление", "зачисление",
|
|
||||||
),
|
|
||||||
"Direction": (
|
|
||||||
"direction", "type", "drcr", "operation type", "movement",
|
|
||||||
"məbləğ növü", "əməliyyat növü", "növ",
|
|
||||||
"тип", "тип операции",
|
|
||||||
),
|
|
||||||
"Currency": (
|
|
||||||
"currency", "ccy", "cur",
|
|
||||||
"valyuta",
|
|
||||||
"валюта",
|
|
||||||
),
|
|
||||||
"Purpose": (
|
|
||||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
|
||||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
|
||||||
"назначение", "назначение платежа", "основание",
|
|
||||||
),
|
|
||||||
"Description": (
|
|
||||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
|
||||||
"qeyd", "şərh", "izahat",
|
|
||||||
"комментарий", "примечание", "описание",
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
_AZERI_TO_LATIN = str.maketrans({
|
|
||||||
"ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o",
|
|
||||||
"ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c",
|
|
||||||
"ğ": "g", "Ğ": "g",
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _norm_for_match(s):
|
|
||||||
if s is None:
|
|
||||||
return ""
|
|
||||||
s = str(s).strip().lower().translate(_AZERI_TO_LATIN)
|
|
||||||
return " ".join(s.split())
|
|
||||||
|
|
||||||
|
|
||||||
def _guess_standard_field(header):
|
|
||||||
"""Return a Standard Field label for the given header text, or None."""
|
|
||||||
h = _norm_for_match(header)
|
|
||||||
if not h:
|
|
||||||
return None
|
|
||||||
# 1. Exact match first.
|
|
||||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
|
||||||
for syn in synonyms:
|
|
||||||
if h == _norm_for_match(syn):
|
|
||||||
return field
|
|
||||||
# 2. Substring fallback: header *contains* a synonym (e.g. "Payment Reference No.").
|
|
||||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
|
||||||
for syn in synonyms:
|
|
||||||
syn_n = _norm_for_match(syn)
|
|
||||||
if syn_n and len(syn_n) >= 4 and syn_n in h:
|
|
||||||
return field
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
@frappe.whitelist()
|
@frappe.whitelist()
|
||||||
def auto_detect_column_mappings(file_url, header_row=1):
|
def auto_detect_column_mappings(file_url, header_row=1):
|
||||||
"""Read sample file headers and propose a mapping to Standard Fields.
|
"""Read sample file headers and propose a mapping to Standard Fields.
|
||||||
|
|
@ -503,6 +400,8 @@ def auto_detect_column_mappings(file_url, header_row=1):
|
||||||
if not headers_result.get("success"):
|
if not headers_result.get("success"):
|
||||||
return headers_result
|
return headers_result
|
||||||
|
|
||||||
|
from jey_erp.bank_integration.header_synonyms import guess_standard_field
|
||||||
|
|
||||||
headers = headers_result.get("headers") or []
|
headers = headers_result.get("headers") or []
|
||||||
mappings = []
|
mappings = []
|
||||||
unmatched = []
|
unmatched = []
|
||||||
|
|
@ -511,7 +410,7 @@ def auto_detect_column_mappings(file_url, header_row=1):
|
||||||
# the loop so both can be picked up.)
|
# the loop so both can be picked up.)
|
||||||
used = set()
|
used = set()
|
||||||
for h in headers:
|
for h in headers:
|
||||||
guess = _guess_standard_field(h)
|
guess = guess_standard_field(h)
|
||||||
if guess and guess not in used:
|
if guess and guess not in used:
|
||||||
mappings.append({"excel_column": h, "standard_field": guess})
|
mappings.append({"excel_column": h, "standard_field": guess})
|
||||||
# Date / Reference / Amount / etc. should appear only once; Debit
|
# Date / Reference / Amount / etc. should appear only once; Debit
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,124 @@
|
||||||
|
"""Header-to-StandardField heuristic dictionary + matcher.
|
||||||
|
|
||||||
|
Used by `excel_parser.auto_detect_column_mappings` to guess a column mapping
|
||||||
|
when the user uploads a sample bank statement. Kept in its own module so the
|
||||||
|
synonym lists can be edited (or extended per-locale) without touching the
|
||||||
|
parser.
|
||||||
|
|
||||||
|
Adding a new synonym: append to the appropriate tuple in HEADER_SYNONYMS.
|
||||||
|
Adding a new language: just add more entries — strings are lower-cased and
|
||||||
|
Azərbaycan-specific letters are transliterated to Latin before matching, so
|
||||||
|
mixed-case / mixed-script entries are fine.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# Synonyms are lowercased; the matcher normalises whitespace and Azərbaycan
|
||||||
|
# letters to Latin equivalents before comparing, so case and the ə/ş/ç/ğ
|
||||||
|
# letters don't matter here.
|
||||||
|
#
|
||||||
|
# Order WITHIN each tuple does not matter — first hit wins for the column.
|
||||||
|
# Order BETWEEN standard fields matters when two fields share a synonym
|
||||||
|
# (e.g. "description" appears in both Purpose and Description's natural-
|
||||||
|
# language pool): the dict order decides which wins. Purpose is listed
|
||||||
|
# before Description so a bank using "Purpose / Description" in one column
|
||||||
|
# lands on Purpose, which is the more useful classification downstream.
|
||||||
|
HEADER_SYNONYMS = {
|
||||||
|
"Date": (
|
||||||
|
"date", "operation date", "transaction date", "value date", "posting date",
|
||||||
|
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||||
|
"дата", "дата операции", "дата документа",
|
||||||
|
),
|
||||||
|
"Reference Number": (
|
||||||
|
"reference", "reference number", "ref", "ref no", "ref number",
|
||||||
|
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||||
|
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||||
|
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
||||||
|
),
|
||||||
|
"Counterparty": (
|
||||||
|
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||||
|
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||||
|
"контрагент", "наименование", "плательщик", "получатель",
|
||||||
|
),
|
||||||
|
"Counterparty Tax ID (VOEN)": (
|
||||||
|
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||||
|
"müştəri voen", "qarşı tərəf voen",
|
||||||
|
"инн", "налоговый номер",
|
||||||
|
),
|
||||||
|
"Counterparty IBAN": (
|
||||||
|
"iban", "counterparty iban", "client iban", "client account",
|
||||||
|
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||||
|
"счёт", "счет", "номер счёта", "номер счета",
|
||||||
|
),
|
||||||
|
"Amount": (
|
||||||
|
"amount", "sum", "total", "value",
|
||||||
|
"məbləğ",
|
||||||
|
"сумма",
|
||||||
|
),
|
||||||
|
"Debit": (
|
||||||
|
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||||
|
"məxaric", "ödəniş",
|
||||||
|
"дебет", "расход", "списание",
|
||||||
|
),
|
||||||
|
"Credit": (
|
||||||
|
"credit", "deposit", "receive", "inflow", "kredit",
|
||||||
|
"mədaxil", "daxilolma",
|
||||||
|
"кредит", "приход", "поступление", "зачисление",
|
||||||
|
),
|
||||||
|
"Direction": (
|
||||||
|
"direction", "type", "drcr", "operation type", "movement",
|
||||||
|
"məbləğ növü", "əməliyyat növü", "növ",
|
||||||
|
"тип", "тип операции",
|
||||||
|
),
|
||||||
|
"Currency": (
|
||||||
|
"currency", "ccy", "cur",
|
||||||
|
"valyuta",
|
||||||
|
"валюта",
|
||||||
|
),
|
||||||
|
"Purpose": (
|
||||||
|
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||||
|
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||||
|
"назначение", "назначение платежа", "основание",
|
||||||
|
),
|
||||||
|
"Description": (
|
||||||
|
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||||
|
"qeyd", "şərh", "izahat",
|
||||||
|
"комментарий", "примечание", "описание",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_AZERI_TO_LATIN = str.maketrans({
|
||||||
|
"ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o",
|
||||||
|
"ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c",
|
||||||
|
"ğ": "g", "Ğ": "g",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _norm(s):
|
||||||
|
if s is None:
|
||||||
|
return ""
|
||||||
|
s = str(s).strip().lower().translate(_AZERI_TO_LATIN)
|
||||||
|
return " ".join(s.split())
|
||||||
|
|
||||||
|
|
||||||
|
def guess_standard_field(header):
|
||||||
|
"""Return a Standard Field label for the given header text, or None.
|
||||||
|
|
||||||
|
Two-pass: exact normalised match first, then substring fallback (catches
|
||||||
|
headers like "Payment Reference No." against the synonym "reference"). The
|
||||||
|
substring pass requires a minimum length of 4 to avoid noise from very
|
||||||
|
short synonyms.
|
||||||
|
"""
|
||||||
|
h = _norm(header)
|
||||||
|
if not h:
|
||||||
|
return None
|
||||||
|
for field, synonyms in HEADER_SYNONYMS.items():
|
||||||
|
for syn in synonyms:
|
||||||
|
if h == _norm(syn):
|
||||||
|
return field
|
||||||
|
for field, synonyms in HEADER_SYNONYMS.items():
|
||||||
|
for syn in synonyms:
|
||||||
|
syn_n = _norm(syn)
|
||||||
|
if syn_n and len(syn_n) >= 4 and syn_n in h:
|
||||||
|
return field
|
||||||
|
return None
|
||||||
Loading…
Reference in New Issue