fix(bank-integration): symmetric Azeri translit + VOEN-first dedup
Two related correctness fixes:
1. Asymmetric Azerbaijani normalization.
consider_azeri_chars was only honoured by match_similar_customers/suppliers
when populating registries. At Create & Reconcile time both the party-name
lookup and the fuzzy purpose matcher compared raw lowercased strings, so a
party matched via "Şirkət" ↔ "Sirket" translit at registry time would not
resolve when the BT arrived, and a purpose rule keyed on "Mədaxil" would
fall short of fuzzy threshold against "Medaxil".
- name_to_party now indexes each row under strict / lowercase / translit /
translit+lowercase variants based on its effective Case Mode + Azeri
Translit (new per-row Select fields that override the globals, same
pattern as case sensitivity).
- _find_mapping_for_txn takes an `az` flag and applies translit to both
keyword cache and txn purpose/party before comparison.
2. Customer / Supplier dedup ignored VOEN.
create_unmapped_customers / create_unmapped_suppliers checked only by
customer_name / supplier_name, so a registry record sharing a VOEN with an
existing party under a slightly different name would create a duplicate.
Now: tax_id is checked first, name as fallback; when an existing party is
found, the mapping row is linked to it (instead of silently `continue`ing
without linking). Response includes linked_count alongside created_count.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
70215e200c
commit
a3ae46b978
|
|
@ -16,6 +16,14 @@ from frappe.utils import cint, flt
|
|||
|
||||
PURPOSE_DOCTYPE = "Bank Integration Purpose"
|
||||
|
||||
# Same translit table as jey_erp.bank_integration.matching._AZERI_MAP — kept
|
||||
# inline here so this module doesn't depend on matching.py at import time.
|
||||
_AZERI_MAP = str.maketrans("ƏəÜüÖöĞğİıÇ窺", "EeUuOoGgIiCcSs")
|
||||
|
||||
|
||||
def _translit_az(text):
|
||||
return (text or "").translate(_AZERI_MAP)
|
||||
|
||||
|
||||
def _partial_ratio(needle, haystack):
|
||||
"""Best fuzzy ratio of `needle` against any equal-length window of `haystack`
|
||||
|
|
@ -49,11 +57,7 @@ def _extract_messages(message_log):
|
|||
|
||||
|
||||
def _effective_case_insensitive(row, global_default):
|
||||
"""Per-row Case Mode overrides the global Ignore Case flag.
|
||||
|
||||
Returns True if the party name for this row should be indexed/looked-up
|
||||
case-insensitively, False otherwise.
|
||||
"""
|
||||
"""Per-row Case Mode overrides the global Ignore Case flag."""
|
||||
mode = (getattr(row, "case_mode", None) or "").strip()
|
||||
if mode == "Ignore Case":
|
||||
return True
|
||||
|
|
@ -62,50 +66,85 @@ def _effective_case_insensitive(row, global_default):
|
|||
return global_default
|
||||
|
||||
|
||||
def _build_name_to_party(settings):
|
||||
"""Counterparty text -> {"Customer": erp_customer} / {"Supplier": erp_supplier},
|
||||
built from the mapped rows of customer_mappings / supplier_mappings.
|
||||
def _effective_azeri(row, global_default):
|
||||
"""Per-row Azeri Translit mode overrides the global Consider Azerbaijani Characters flag."""
|
||||
mode = (getattr(row, "azeri_mode", None) or "").strip()
|
||||
if mode == "Apply Translit":
|
||||
return True
|
||||
if mode == "Strict (No Translit)":
|
||||
return False
|
||||
return global_default
|
||||
|
||||
A row may be indexed under its strict key only, its lowercased key only, or
|
||||
both — depending on the row's Case Mode (which overrides the global
|
||||
`case_insensitive_party_match` setting).
|
||||
|
||||
def _build_name_to_party(settings):
|
||||
"""Counterparty text -> {"Customer": erp_customer} / {"Supplier": erp_supplier}.
|
||||
|
||||
Each row is indexed under up to four key variants depending on its effective
|
||||
Case Mode and Azeri Translit settings (both of which fall back to globals):
|
||||
strict : original `.strip()` key
|
||||
lowercase : when case-insensitive
|
||||
translit : when azeri-translit
|
||||
translit + lower : when both
|
||||
"""
|
||||
global_ci = bool(getattr(settings, "case_insensitive_party_match", 1))
|
||||
global_az = bool(getattr(settings, "consider_azeri_chars", 1))
|
||||
idx = {}
|
||||
|
||||
def _add(name, party_type, erp_party, ci):
|
||||
def _add(name, party_type, erp_party, ci, az):
|
||||
key = (name or "").strip()
|
||||
if not key:
|
||||
return
|
||||
idx.setdefault(key, {})[party_type] = erp_party
|
||||
if ci:
|
||||
idx.setdefault(key.lower(), {})[party_type] = erp_party
|
||||
if az:
|
||||
translit = _translit_az(key)
|
||||
if translit != key:
|
||||
idx.setdefault(translit, {})[party_type] = erp_party
|
||||
if ci:
|
||||
idx.setdefault(translit.lower(), {})[party_type] = erp_party
|
||||
|
||||
for row in settings.customer_mappings:
|
||||
if not row.erp_customer or not row.bi_customer_name:
|
||||
continue
|
||||
cust_name = frappe.db.get_value("Bank Integration Customer", row.bi_customer_name, "customer_name")
|
||||
if cust_name:
|
||||
_add(cust_name, "Customer", row.erp_customer, _effective_case_insensitive(row, global_ci))
|
||||
_add(
|
||||
cust_name, "Customer", row.erp_customer,
|
||||
_effective_case_insensitive(row, global_ci),
|
||||
_effective_azeri(row, global_az),
|
||||
)
|
||||
for row in settings.supplier_mappings:
|
||||
if not row.erp_supplier or not row.bi_supplier_name:
|
||||
continue
|
||||
supp_name = frappe.db.get_value("Bank Integration Supplier", row.bi_supplier_name, "supplier_name")
|
||||
if supp_name:
|
||||
_add(supp_name, "Supplier", row.erp_supplier, _effective_case_insensitive(row, global_ci))
|
||||
_add(
|
||||
supp_name, "Supplier", row.erp_supplier,
|
||||
_effective_case_insensitive(row, global_ci),
|
||||
_effective_azeri(row, global_az),
|
||||
)
|
||||
return idx
|
||||
|
||||
|
||||
def _resolve_party_from_name(name, name_to_party, payment_type):
|
||||
"""(party_type, erp_party) for `name`, or (None, None). Tries an exact (strip-only)
|
||||
lookup first; falls back to lowercased lookup so rows indexed case-insensitively
|
||||
can be reached from arbitrary Bank Transaction bank_party_name casing.
|
||||
"""(party_type, erp_party) for `name`, or (None, None).
|
||||
|
||||
Tries up to four lookup variants — strict, lowercase, translit, translit+lower.
|
||||
The index encodes per-row decisions, so we don't need flags here.
|
||||
Prefers Customer for incoming (Receive) and Supplier for outgoing (Pay),
|
||||
falling back to the other."""
|
||||
falling back to the other.
|
||||
"""
|
||||
key = (name or "").strip()
|
||||
if not key:
|
||||
return None, None
|
||||
cands = name_to_party.get(key) or name_to_party.get(key.lower())
|
||||
translit = _translit_az(key)
|
||||
cands = (
|
||||
name_to_party.get(key)
|
||||
or name_to_party.get(key.lower())
|
||||
or (name_to_party.get(translit) if translit != key else None)
|
||||
or (name_to_party.get(translit.lower()) if translit != key else None)
|
||||
)
|
||||
if not cands:
|
||||
return None, None
|
||||
preferred = "Customer" if payment_type == "Receive" else "Supplier"
|
||||
|
|
@ -256,12 +295,13 @@ def create_purpose_mappings(transactions, bank_integration, paid_from=None, paid
|
|||
|
||||
if do_documents:
|
||||
purpose_threshold = flt(settings.similarity_threshold_purpose or 70) / 100.0
|
||||
az = bool(getattr(settings, "consider_azeri_chars", 1))
|
||||
if mode == "Documents & Reconcile":
|
||||
purpose_text_cache = {}
|
||||
for row in settings.transaction_mappings:
|
||||
if row.purpose_keyword and row.purpose_keyword not in purpose_text_cache:
|
||||
text = frappe.db.get_value(PURPOSE_DOCTYPE, row.purpose_keyword, "purpose_keyword") or ""
|
||||
purpose_text_cache[row.purpose_keyword] = text.lower()
|
||||
purpose_text_cache[row.purpose_keyword] = _norm_text(text, az)
|
||||
txn_mappings = settings.transaction_mappings
|
||||
else:
|
||||
purpose_text_cache = {}
|
||||
|
|
@ -276,7 +316,7 @@ def create_purpose_mappings(transactions, bank_integration, paid_from=None, paid
|
|||
continue
|
||||
|
||||
if mode == "Documents & Reconcile":
|
||||
mapping_row = _find_mapping_for_txn(txn, txn_mappings, purpose_text_cache, purpose_threshold)
|
||||
mapping_row = _find_mapping_for_txn(txn, txn_mappings, purpose_text_cache, purpose_threshold, az)
|
||||
if not mapping_row:
|
||||
errors.append({
|
||||
"reference_number": txn.get("reference_number") or "",
|
||||
|
|
@ -378,17 +418,28 @@ def create_purpose_mappings(transactions, bank_integration, paid_from=None, paid
|
|||
return {"success": False, "message": str(e)}
|
||||
|
||||
|
||||
def _find_mapping_for_txn(txn, transaction_mappings, purpose_text_cache, purpose_threshold):
|
||||
def _norm_text(text, az):
|
||||
"""Lowercased + (optionally) Azeri-translit form for purpose/party comparison."""
|
||||
s = (text or "").lower()
|
||||
if az:
|
||||
s = _translit_az(s)
|
||||
return s
|
||||
|
||||
|
||||
def _find_mapping_for_txn(txn, transaction_mappings, purpose_text_cache, purpose_threshold, az=False):
|
||||
"""Find the best matching transaction mapping row for a BRT txn.
|
||||
|
||||
Priority: purpose+counterparty (exact) > purpose-only (exact) >
|
||||
purpose+counterparty (fuzzy) > purpose-only (fuzzy) > counterparty-only >
|
||||
fallback by counterparty_type. "Exact" = the keyword is a substring of the
|
||||
transaction's description; "fuzzy" = partial-ratio ≥ purpose_threshold.
|
||||
|
||||
`az` toggles Azerbaijani transliteration of purpose / party text before
|
||||
comparison (driven by settings.consider_azeri_chars).
|
||||
"""
|
||||
payment_type = "Pay" if txn.get("drcr") == "D" else "Receive"
|
||||
txn_purpose = (txn.get("purpose") or "").lower()
|
||||
txn_party = (txn.get("party") or "").lower()
|
||||
txn_purpose = _norm_text(txn.get("purpose"), az)
|
||||
txn_party = _norm_text(txn.get("party"), az)
|
||||
|
||||
bt_name = txn.get("bank_transaction_name")
|
||||
txn_currency = (frappe.db.get_value("Bank Transaction", bt_name, "currency") or "").strip().upper() if bt_name else ""
|
||||
|
|
@ -431,7 +482,7 @@ def _find_mapping_for_txn(txn, transaction_mappings, purpose_text_cache, purpose
|
|||
return _partial_ratio(kw, txn_purpose) if kw else 0.0
|
||||
|
||||
def _counterparty_matches(row):
|
||||
return row.counterparty.lower() == txn_party
|
||||
return _norm_text(row.counterparty, az) == txn_party
|
||||
|
||||
def _best_fuzzy(rules, check_cp):
|
||||
best = None
|
||||
|
|
|
|||
|
|
@ -4,6 +4,24 @@ import frappe
|
|||
from frappe import _
|
||||
|
||||
|
||||
def _find_existing_party(doctype, name_field, name, tax_id):
|
||||
"""VOEN-first lookup: a matching tax_id wins over a name match.
|
||||
|
||||
Returns the existing party's name, or None.
|
||||
"""
|
||||
tid = (tax_id or "").strip()
|
||||
if tid:
|
||||
existing = frappe.db.get_value(doctype, {"tax_id": tid}, "name")
|
||||
if existing:
|
||||
return existing
|
||||
clean_name = (name or "").strip()
|
||||
if clean_name:
|
||||
existing = frappe.db.get_value(doctype, {name_field: clean_name}, "name")
|
||||
if existing:
|
||||
return existing
|
||||
return None
|
||||
|
||||
|
||||
def _pick_leaf(doctype, configured, label):
|
||||
"""Validate `configured` as a leaf node of an NSM tree doctype
|
||||
(Customer Group / Supplier Group / Territory).
|
||||
|
|
@ -47,10 +65,29 @@ def create_unmapped_customers(bank_integration):
|
|||
]
|
||||
|
||||
created = 0
|
||||
linked = 0
|
||||
for (mapping, bi_cust), (customer_group, territory) in zip(unmapped, resolved):
|
||||
try:
|
||||
customer_name = bi_cust.customer_name
|
||||
if frappe.db.exists("Customer", {"customer_name": customer_name}):
|
||||
payment_terms = mapping.payment_terms or bi.default_payment_terms
|
||||
|
||||
# VOEN beats name: if an existing Customer shares this tax_id, reuse it
|
||||
# rather than creating a duplicate. Same for name as a fallback.
|
||||
existing = _find_existing_party("Customer", "customer_name", customer_name, bi_cust.tax_id)
|
||||
if existing:
|
||||
mapping.erp_customer = existing
|
||||
mapping.mapping_type = "Manual"
|
||||
existing_group, existing_territory = frappe.db.get_value(
|
||||
"Customer", existing, ["customer_group", "territory"]
|
||||
) or (None, None)
|
||||
frappe.db.set_value("Bank Integration Customer", bi_cust.name, {
|
||||
"status": "Mapped",
|
||||
"mapped_customer": existing,
|
||||
"customer_group": existing_group,
|
||||
"territory": existing_territory,
|
||||
"payment_terms": payment_terms or None,
|
||||
}, update_modified=False)
|
||||
linked += 1
|
||||
continue
|
||||
|
||||
party = frappe.new_doc("Customer")
|
||||
|
|
@ -62,7 +99,6 @@ def create_unmapped_customers(bank_integration):
|
|||
if bi_cust.tax_id:
|
||||
party.tax_id = bi_cust.tax_id
|
||||
|
||||
payment_terms = mapping.payment_terms or bi.default_payment_terms
|
||||
if payment_terms:
|
||||
party.payment_terms = payment_terms
|
||||
|
||||
|
|
@ -86,10 +122,10 @@ def create_unmapped_customers(bank_integration):
|
|||
"BI Customer Creation",
|
||||
)
|
||||
|
||||
if created > 0:
|
||||
if created > 0 or linked > 0:
|
||||
bi.save(ignore_permissions=True)
|
||||
frappe.db.commit()
|
||||
return {"success": True, "created_count": created}
|
||||
return {"success": True, "created_count": created, "linked_count": linked}
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
|
|
@ -112,10 +148,24 @@ def create_unmapped_suppliers(bank_integration):
|
|||
]
|
||||
|
||||
created = 0
|
||||
linked = 0
|
||||
for (mapping, bi_supp), supplier_group in zip(unmapped, resolved):
|
||||
try:
|
||||
supplier_name = bi_supp.supplier_name
|
||||
if frappe.db.exists("Supplier", {"supplier_name": supplier_name}):
|
||||
payment_terms = mapping.payment_terms or bi.default_payment_terms
|
||||
|
||||
existing = _find_existing_party("Supplier", "supplier_name", supplier_name, bi_supp.tax_id)
|
||||
if existing:
|
||||
mapping.erp_supplier = existing
|
||||
mapping.mapping_type = "Manual"
|
||||
existing_group = frappe.db.get_value("Supplier", existing, "supplier_group")
|
||||
frappe.db.set_value("Bank Integration Supplier", bi_supp.name, {
|
||||
"status": "Mapped",
|
||||
"mapped_supplier": existing,
|
||||
"supplier_group": existing_group,
|
||||
"payment_terms": payment_terms or None,
|
||||
}, update_modified=False)
|
||||
linked += 1
|
||||
continue
|
||||
|
||||
party = frappe.new_doc("Supplier")
|
||||
|
|
@ -126,7 +176,6 @@ def create_unmapped_suppliers(bank_integration):
|
|||
if bi_supp.tax_id:
|
||||
party.tax_id = bi_supp.tax_id
|
||||
|
||||
payment_terms = mapping.payment_terms or bi.default_payment_terms
|
||||
if payment_terms:
|
||||
party.payment_terms = payment_terms
|
||||
|
||||
|
|
@ -149,10 +198,10 @@ def create_unmapped_suppliers(bank_integration):
|
|||
"BI Supplier Creation",
|
||||
)
|
||||
|
||||
if created > 0:
|
||||
if created > 0 or linked > 0:
|
||||
bi.save(ignore_permissions=True)
|
||||
frappe.db.commit()
|
||||
return {"success": True, "created_count": created}
|
||||
return {"success": True, "created_count": created, "linked_count": linked}
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"territory",
|
||||
"payment_terms",
|
||||
"case_mode",
|
||||
"azeri_mode",
|
||||
"mapping_type"
|
||||
],
|
||||
"fields": [
|
||||
|
|
@ -63,6 +64,13 @@
|
|||
"label": "Case Mode",
|
||||
"options": "\nIgnore Case\nCase Sensitive"
|
||||
},
|
||||
{
|
||||
"description": "Override the global 'Consider Azerbaijani Characters' for this row's party-name matching. Blank = use the global setting.",
|
||||
"fieldname": "azeri_mode",
|
||||
"fieldtype": "Select",
|
||||
"label": "Azeri Translit",
|
||||
"options": "\nApply Translit\nStrict (No Translit)"
|
||||
},
|
||||
{
|
||||
"default": "Manual",
|
||||
"fieldname": "mapping_type",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"supplier_group",
|
||||
"payment_terms",
|
||||
"case_mode",
|
||||
"azeri_mode",
|
||||
"mapping_type"
|
||||
],
|
||||
"fields": [
|
||||
|
|
@ -55,6 +56,13 @@
|
|||
"label": "Case Mode",
|
||||
"options": "\nIgnore Case\nCase Sensitive"
|
||||
},
|
||||
{
|
||||
"description": "Override the global 'Consider Azerbaijani Characters' for this row's party-name matching. Blank = use the global setting.",
|
||||
"fieldname": "azeri_mode",
|
||||
"fieldtype": "Select",
|
||||
"label": "Azeri Translit",
|
||||
"options": "\nApply Translit\nStrict (No Translit)"
|
||||
},
|
||||
{
|
||||
"default": "Manual",
|
||||
"fieldname": "mapping_type",
|
||||
|
|
|
|||
Loading…
Reference in New Issue