refactor(synonyms): move synonym data to JSON, keep matcher in Python
The header → Standard Field dictionary is data, not code, so it now lives in header_synonyms.json next to its loader. Editing a synonym list no longer requires touching Python — open the JSON, append a string, done. header_synonyms.py is now a thin loader (reads the JSON once at import time) plus the matcher and the Azərbaycan→Latin transliteration table, which stay in Python because they're behaviour, not data.
This commit is contained in:
parent
357305a687
commit
ded4946c75
|
|
@ -0,0 +1,65 @@
|
|||
{
|
||||
"_comment": "Header text → Standard Field synonyms used by auto-detect when a sample bank statement is attached. Strings are lower-cased and Azərbaycan letters (ə/ş/ç/ğ/ı/İ/ö/ü) are transliterated to Latin before matching, so case and script don't matter here. To add a new synonym just append a string to the right list. To add a new language add more strings — no code change needed. Order WITHIN a list is irrelevant; order BETWEEN keys is the tie-breaker when two fields share a synonym (e.g. Purpose vs Description) — earlier key wins.",
|
||||
|
||||
"Date": [
|
||||
"date", "operation date", "transaction date", "value date", "posting date",
|
||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||
"дата", "дата операции", "дата документа"
|
||||
],
|
||||
"Reference Number": [
|
||||
"reference", "reference number", "ref", "ref no", "ref number",
|
||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||
"номер документа", "номер", "док", "ссылка", "идентификатор"
|
||||
],
|
||||
"Counterparty": [
|
||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||
"контрагент", "наименование", "плательщик", "получатель"
|
||||
],
|
||||
"Counterparty Tax ID (VOEN)": [
|
||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||
"müştəri voen", "qarşı tərəf voen",
|
||||
"инн", "налоговый номер"
|
||||
],
|
||||
"Counterparty IBAN": [
|
||||
"iban", "counterparty iban", "client iban", "client account",
|
||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||
"счёт", "счет", "номер счёта", "номер счета"
|
||||
],
|
||||
"Amount": [
|
||||
"amount", "sum", "total", "value",
|
||||
"məbləğ",
|
||||
"сумма"
|
||||
],
|
||||
"Debit": [
|
||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||
"məxaric", "ödəniş",
|
||||
"дебет", "расход", "списание"
|
||||
],
|
||||
"Credit": [
|
||||
"credit", "deposit", "receive", "inflow", "kredit",
|
||||
"mədaxil", "daxilolma",
|
||||
"кредит", "приход", "поступление", "зачисление"
|
||||
],
|
||||
"Direction": [
|
||||
"direction", "type", "drcr", "operation type", "movement",
|
||||
"məbləğ növü", "əməliyyat növü", "növ",
|
||||
"тип", "тип операции"
|
||||
],
|
||||
"Currency": [
|
||||
"currency", "ccy", "cur",
|
||||
"valyuta",
|
||||
"валюта"
|
||||
],
|
||||
"Purpose": [
|
||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||
"назначение", "назначение платежа", "основание"
|
||||
],
|
||||
"Description": [
|
||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||
"qeyd", "şərh", "izahat",
|
||||
"комментарий", "примечание", "описание"
|
||||
]
|
||||
}
|
||||
|
|
@ -1,90 +1,25 @@
|
|||
"""Header-to-StandardField heuristic dictionary + matcher.
|
||||
"""Loader + matcher for the Header → Standard Field synonym dictionary.
|
||||
|
||||
Used by `excel_parser.auto_detect_column_mappings` to guess a column mapping
|
||||
when the user uploads a sample bank statement. Kept in its own module so the
|
||||
synonym lists can be edited (or extended per-locale) without touching the
|
||||
parser.
|
||||
|
||||
Adding a new synonym: append to the appropriate tuple in HEADER_SYNONYMS.
|
||||
Adding a new language: just add more entries — strings are lower-cased and
|
||||
Azərbaycan-specific letters are transliterated to Latin before matching, so
|
||||
mixed-case / mixed-script entries are fine.
|
||||
The actual synonym data lives next to this file in `header_synonyms.json`
|
||||
so it can be edited without touching Python code.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
# Synonyms are lowercased; the matcher normalises whitespace and Azərbaycan
|
||||
# letters to Latin equivalents before comparing, so case and the ə/ş/ç/ğ
|
||||
# letters don't matter here.
|
||||
#
|
||||
# Order WITHIN each tuple does not matter — first hit wins for the column.
|
||||
# Order BETWEEN standard fields matters when two fields share a synonym
|
||||
# (e.g. "description" appears in both Purpose and Description's natural-
|
||||
# language pool): the dict order decides which wins. Purpose is listed
|
||||
# before Description so a bank using "Purpose / Description" in one column
|
||||
# lands on Purpose, which is the more useful classification downstream.
|
||||
HEADER_SYNONYMS = {
|
||||
"Date": (
|
||||
"date", "operation date", "transaction date", "value date", "posting date",
|
||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||
"дата", "дата операции", "дата документа",
|
||||
),
|
||||
"Reference Number": (
|
||||
"reference", "reference number", "ref", "ref no", "ref number",
|
||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
||||
),
|
||||
"Counterparty": (
|
||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||
"контрагент", "наименование", "плательщик", "получатель",
|
||||
),
|
||||
"Counterparty Tax ID (VOEN)": (
|
||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||
"müştəri voen", "qarşı tərəf voen",
|
||||
"инн", "налоговый номер",
|
||||
),
|
||||
"Counterparty IBAN": (
|
||||
"iban", "counterparty iban", "client iban", "client account",
|
||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||
"счёт", "счет", "номер счёта", "номер счета",
|
||||
),
|
||||
"Amount": (
|
||||
"amount", "sum", "total", "value",
|
||||
"məbləğ",
|
||||
"сумма",
|
||||
),
|
||||
"Debit": (
|
||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||
"məxaric", "ödəniş",
|
||||
"дебет", "расход", "списание",
|
||||
),
|
||||
"Credit": (
|
||||
"credit", "deposit", "receive", "inflow", "kredit",
|
||||
"mədaxil", "daxilolma",
|
||||
"кредит", "приход", "поступление", "зачисление",
|
||||
),
|
||||
"Direction": (
|
||||
"direction", "type", "drcr", "operation type", "movement",
|
||||
"məbləğ növü", "əməliyyat növü", "növ",
|
||||
"тип", "тип операции",
|
||||
),
|
||||
"Currency": (
|
||||
"currency", "ccy", "cur",
|
||||
"valyuta",
|
||||
"валюта",
|
||||
),
|
||||
"Purpose": (
|
||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||
"назначение", "назначение платежа", "основание",
|
||||
),
|
||||
"Description": (
|
||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||
"qeyd", "şərh", "izahat",
|
||||
"комментарий", "примечание", "описание",
|
||||
),
|
||||
}
|
||||
|
||||
_HERE = os.path.dirname(__file__)
|
||||
_JSON_PATH = os.path.join(_HERE, "header_synonyms.json")
|
||||
|
||||
|
||||
def _load_synonyms():
|
||||
with open(_JSON_PATH, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
# Drop the leading "_comment" key (used as inline documentation in the JSON).
|
||||
return {k: tuple(v) for k, v in data.items() if not k.startswith("_") and isinstance(v, list)}
|
||||
|
||||
|
||||
HEADER_SYNONYMS = _load_synonyms()
|
||||
|
||||
|
||||
_AZERI_TO_LATIN = str.maketrans({
|
||||
|
|
|
|||
Loading…
Reference in New Issue