refactor(synonyms): move synonym data to JSON, keep matcher in Python
The header → Standard Field dictionary is data, not code, so it now lives in header_synonyms.json next to its loader. Editing a synonym list no longer requires touching Python — open the JSON, append a string, done. header_synonyms.py is now a thin loader (reads the JSON once at import time) plus the matcher and the Azərbaycan→Latin transliteration table, which stay in Python because they're behaviour, not data.
This commit is contained in:
parent
357305a687
commit
ded4946c75
|
|
@ -0,0 +1,65 @@
|
||||||
|
{
|
||||||
|
"_comment": "Header text → Standard Field synonyms used by auto-detect when a sample bank statement is attached. Strings are lower-cased and Azərbaycan letters (ə/ş/ç/ğ/ı/İ/ö/ü) are transliterated to Latin before matching, so case and script don't matter here. To add a new synonym just append a string to the right list. To add a new language add more strings — no code change needed. Order WITHIN a list is irrelevant; order BETWEEN keys is the tie-breaker when two fields share a synonym (e.g. Purpose vs Description) — earlier key wins.",
|
||||||
|
|
||||||
|
"Date": [
|
||||||
|
"date", "operation date", "transaction date", "value date", "posting date",
|
||||||
|
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||||
|
"дата", "дата операции", "дата документа"
|
||||||
|
],
|
||||||
|
"Reference Number": [
|
||||||
|
"reference", "reference number", "ref", "ref no", "ref number",
|
||||||
|
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||||
|
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||||
|
"номер документа", "номер", "док", "ссылка", "идентификатор"
|
||||||
|
],
|
||||||
|
"Counterparty": [
|
||||||
|
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||||
|
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||||
|
"контрагент", "наименование", "плательщик", "получатель"
|
||||||
|
],
|
||||||
|
"Counterparty Tax ID (VOEN)": [
|
||||||
|
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||||
|
"müştəri voen", "qarşı tərəf voen",
|
||||||
|
"инн", "налоговый номер"
|
||||||
|
],
|
||||||
|
"Counterparty IBAN": [
|
||||||
|
"iban", "counterparty iban", "client iban", "client account",
|
||||||
|
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||||
|
"счёт", "счет", "номер счёта", "номер счета"
|
||||||
|
],
|
||||||
|
"Amount": [
|
||||||
|
"amount", "sum", "total", "value",
|
||||||
|
"məbləğ",
|
||||||
|
"сумма"
|
||||||
|
],
|
||||||
|
"Debit": [
|
||||||
|
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||||
|
"məxaric", "ödəniş",
|
||||||
|
"дебет", "расход", "списание"
|
||||||
|
],
|
||||||
|
"Credit": [
|
||||||
|
"credit", "deposit", "receive", "inflow", "kredit",
|
||||||
|
"mədaxil", "daxilolma",
|
||||||
|
"кредит", "приход", "поступление", "зачисление"
|
||||||
|
],
|
||||||
|
"Direction": [
|
||||||
|
"direction", "type", "drcr", "operation type", "movement",
|
||||||
|
"məbləğ növü", "əməliyyat növü", "növ",
|
||||||
|
"тип", "тип операции"
|
||||||
|
],
|
||||||
|
"Currency": [
|
||||||
|
"currency", "ccy", "cur",
|
||||||
|
"valyuta",
|
||||||
|
"валюта"
|
||||||
|
],
|
||||||
|
"Purpose": [
|
||||||
|
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||||
|
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||||
|
"назначение", "назначение платежа", "основание"
|
||||||
|
],
|
||||||
|
"Description": [
|
||||||
|
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||||
|
"qeyd", "şərh", "izahat",
|
||||||
|
"комментарий", "примечание", "описание"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -1,90 +1,25 @@
|
||||||
"""Header-to-StandardField heuristic dictionary + matcher.
|
"""Loader + matcher for the Header → Standard Field synonym dictionary.
|
||||||
|
|
||||||
Used by `excel_parser.auto_detect_column_mappings` to guess a column mapping
|
The actual synonym data lives next to this file in `header_synonyms.json`
|
||||||
when the user uploads a sample bank statement. Kept in its own module so the
|
so it can be edited without touching Python code.
|
||||||
synonym lists can be edited (or extended per-locale) without touching the
|
|
||||||
parser.
|
|
||||||
|
|
||||||
Adding a new synonym: append to the appropriate tuple in HEADER_SYNONYMS.
|
|
||||||
Adding a new language: just add more entries — strings are lower-cased and
|
|
||||||
Azərbaycan-specific letters are transliterated to Latin before matching, so
|
|
||||||
mixed-case / mixed-script entries are fine.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
# Synonyms are lowercased; the matcher normalises whitespace and Azərbaycan
|
|
||||||
# letters to Latin equivalents before comparing, so case and the ə/ş/ç/ğ
|
_HERE = os.path.dirname(__file__)
|
||||||
# letters don't matter here.
|
_JSON_PATH = os.path.join(_HERE, "header_synonyms.json")
|
||||||
#
|
|
||||||
# Order WITHIN each tuple does not matter — first hit wins for the column.
|
|
||||||
# Order BETWEEN standard fields matters when two fields share a synonym
|
def _load_synonyms():
|
||||||
# (e.g. "description" appears in both Purpose and Description's natural-
|
with open(_JSON_PATH, encoding="utf-8") as f:
|
||||||
# language pool): the dict order decides which wins. Purpose is listed
|
data = json.load(f)
|
||||||
# before Description so a bank using "Purpose / Description" in one column
|
# Drop the leading "_comment" key (used as inline documentation in the JSON).
|
||||||
# lands on Purpose, which is the more useful classification downstream.
|
return {k: tuple(v) for k, v in data.items() if not k.startswith("_") and isinstance(v, list)}
|
||||||
HEADER_SYNONYMS = {
|
|
||||||
"Date": (
|
|
||||||
"date", "operation date", "transaction date", "value date", "posting date",
|
HEADER_SYNONYMS = _load_synonyms()
|
||||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
|
||||||
"дата", "дата операции", "дата документа",
|
|
||||||
),
|
|
||||||
"Reference Number": (
|
|
||||||
"reference", "reference number", "ref", "ref no", "ref number",
|
|
||||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
|
||||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
|
||||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
|
||||||
),
|
|
||||||
"Counterparty": (
|
|
||||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
|
||||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
|
||||||
"контрагент", "наименование", "плательщик", "получатель",
|
|
||||||
),
|
|
||||||
"Counterparty Tax ID (VOEN)": (
|
|
||||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
|
||||||
"müştəri voen", "qarşı tərəf voen",
|
|
||||||
"инн", "налоговый номер",
|
|
||||||
),
|
|
||||||
"Counterparty IBAN": (
|
|
||||||
"iban", "counterparty iban", "client iban", "client account",
|
|
||||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
|
||||||
"счёт", "счет", "номер счёта", "номер счета",
|
|
||||||
),
|
|
||||||
"Amount": (
|
|
||||||
"amount", "sum", "total", "value",
|
|
||||||
"məbləğ",
|
|
||||||
"сумма",
|
|
||||||
),
|
|
||||||
"Debit": (
|
|
||||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
|
||||||
"məxaric", "ödəniş",
|
|
||||||
"дебет", "расход", "списание",
|
|
||||||
),
|
|
||||||
"Credit": (
|
|
||||||
"credit", "deposit", "receive", "inflow", "kredit",
|
|
||||||
"mədaxil", "daxilolma",
|
|
||||||
"кредит", "приход", "поступление", "зачисление",
|
|
||||||
),
|
|
||||||
"Direction": (
|
|
||||||
"direction", "type", "drcr", "operation type", "movement",
|
|
||||||
"məbləğ növü", "əməliyyat növü", "növ",
|
|
||||||
"тип", "тип операции",
|
|
||||||
),
|
|
||||||
"Currency": (
|
|
||||||
"currency", "ccy", "cur",
|
|
||||||
"valyuta",
|
|
||||||
"валюта",
|
|
||||||
),
|
|
||||||
"Purpose": (
|
|
||||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
|
||||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
|
||||||
"назначение", "назначение платежа", "основание",
|
|
||||||
),
|
|
||||||
"Description": (
|
|
||||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
|
||||||
"qeyd", "şərh", "izahat",
|
|
||||||
"комментарий", "примечание", "описание",
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
_AZERI_TO_LATIN = str.maketrans({
|
_AZERI_TO_LATIN = str.maketrans({
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue