feat(bank-integration): user-defined direction aliases in Mode 3
'Single column + direction column' previously guessed direction from a hardcoded EN/AZ alias list, silently falling back to "D" on anything unrecognised. Now the preset exposes two comma-separated fields — Debit Values and Credit Values — that the user fills with whatever their bank prints (e.g. "DR, Debit, Məxaric" / "CR, Credit, Mədaxil"). The fields are only visible/required for this amount mode and validated on save. Rows with a Direction value that doesn't match either list are now dropped with reason="direction" and the offending values are surfaced in the import / Load Data dialogs so the user can copy them straight into the preset. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
3b1b43a57c
commit
bd5d773daa
|
|
@ -107,40 +107,73 @@ def parse_excel(file_url, preset_name):
|
||||||
use_custom = bool(preset.get("use_custom_date_format"))
|
use_custom = bool(preset.get("use_custom_date_format"))
|
||||||
custom_format = (preset.date_format or "").strip() if use_custom else ""
|
custom_format = (preset.date_format or "").strip() if use_custom else ""
|
||||||
amount_mode = preset.amount_mode or "Separate debit/credit columns"
|
amount_mode = preset.amount_mode or "Separate debit/credit columns"
|
||||||
|
debit_aliases = _split_aliases(preset.get("direction_debit_values"))
|
||||||
|
credit_aliases = _split_aliases(preset.get("direction_credit_values"))
|
||||||
|
|
||||||
|
if amount_mode == "Single column + direction column" and not (debit_aliases or credit_aliases):
|
||||||
|
frappe.throw(
|
||||||
|
"Preset '{}' uses 'Single column + direction column' but has no Debit/Credit values configured. "
|
||||||
|
"Open the preset and fill in at least one of 'Debit Values' / 'Credit Values'.".format(preset_name)
|
||||||
|
)
|
||||||
|
|
||||||
data_start = header_idx + 1
|
data_start = header_idx + 1
|
||||||
|
|
||||||
transactions = []
|
transactions = []
|
||||||
dropped_date = 0
|
dropped_date = 0
|
||||||
dropped_amount = 0
|
dropped_amount = 0
|
||||||
|
dropped_direction = 0
|
||||||
|
unknown_directions = set()
|
||||||
for raw_row in all_rows[data_start:]:
|
for raw_row in all_rows[data_start:]:
|
||||||
if not raw_row or all(c is None or str(c).strip() == "" for c in raw_row):
|
if not raw_row or all(c is None or str(c).strip() == "" for c in raw_row):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
parsed, reason = _parse_row(raw_row, idx_to_standard, custom_format, amount_mode)
|
parsed, reason, ctx = _parse_row(
|
||||||
|
raw_row, idx_to_standard, custom_format, amount_mode,
|
||||||
|
debit_aliases, credit_aliases,
|
||||||
|
)
|
||||||
if parsed:
|
if parsed:
|
||||||
transactions.append(parsed)
|
transactions.append(parsed)
|
||||||
elif reason == "date":
|
elif reason == "date":
|
||||||
dropped_date += 1
|
dropped_date += 1
|
||||||
elif reason == "amount":
|
elif reason == "amount":
|
||||||
dropped_amount += 1
|
dropped_amount += 1
|
||||||
|
elif reason == "direction":
|
||||||
|
dropped_direction += 1
|
||||||
|
if ctx:
|
||||||
|
unknown_directions.add(ctx)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"transactions": transactions,
|
"transactions": transactions,
|
||||||
"dropped_date": dropped_date,
|
"dropped_date": dropped_date,
|
||||||
"dropped_amount": dropped_amount,
|
"dropped_amount": dropped_amount,
|
||||||
|
"dropped_direction": dropped_direction,
|
||||||
|
"unknown_directions": sorted(unknown_directions)[:20],
|
||||||
}
|
}
|
||||||
finally:
|
finally:
|
||||||
wb.close()
|
wb.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _split_aliases(raw):
|
||||||
|
"""Parse a comma-separated alias list from a preset field into an uppercase set."""
|
||||||
|
if not raw:
|
||||||
|
return set()
|
||||||
|
return {part.strip().upper() for part in str(raw).split(",") if part.strip()}
|
||||||
|
|
||||||
|
|
||||||
def _normalize_header(value):
|
def _normalize_header(value):
|
||||||
if value is None:
|
if value is None:
|
||||||
return ""
|
return ""
|
||||||
return " ".join(str(value).strip().lower().split())
|
return " ".join(str(value).strip().lower().split())
|
||||||
|
|
||||||
|
|
||||||
def _parse_row(raw_row, idx_to_standard, custom_format, amount_mode):
|
def _parse_row(raw_row, idx_to_standard, custom_format, amount_mode,
|
||||||
"""Returns (record_dict_or_None, reason). reason in {"", "date", "amount"}."""
|
debit_aliases=None, credit_aliases=None):
|
||||||
|
"""Returns (record_dict_or_None, reason, ctx).
|
||||||
|
|
||||||
|
reason in {"", "date", "amount", "direction"}; ctx is the offending value
|
||||||
|
when applicable (used to show the user which Direction strings were not
|
||||||
|
recognised so they can add them to the preset).
|
||||||
|
"""
|
||||||
record = {
|
record = {
|
||||||
"ref_no": "",
|
"ref_no": "",
|
||||||
"date": None,
|
"date": None,
|
||||||
|
|
@ -201,33 +234,33 @@ def _parse_row(raw_row, idx_to_standard, custom_format, amount_mode):
|
||||||
record["amount"] = abs(flt(credit_val))
|
record["amount"] = abs(flt(credit_val))
|
||||||
record["drcr"] = "C"
|
record["drcr"] = "C"
|
||||||
else:
|
else:
|
||||||
return None, "amount"
|
return None, "amount", None
|
||||||
elif amount_mode == "Single column with sign":
|
elif amount_mode == "Single column with sign":
|
||||||
if amount_val is None:
|
if amount_val is None:
|
||||||
return None, "amount"
|
return None, "amount", None
|
||||||
record["amount"] = abs(flt(amount_val))
|
record["amount"] = abs(flt(amount_val))
|
||||||
record["drcr"] = "D" if amount_val < 0 else "C"
|
record["drcr"] = "D" if amount_val < 0 else "C"
|
||||||
elif amount_mode == "Single column + direction column":
|
elif amount_mode == "Single column + direction column":
|
||||||
if amount_val is None or direction_val is None:
|
if amount_val is None or direction_val is None:
|
||||||
return None, "amount"
|
return None, "amount", None
|
||||||
record["amount"] = abs(flt(amount_val))
|
record["amount"] = abs(flt(amount_val))
|
||||||
if direction_val in ("D", "DR", "DEBIT", "DT", "OUT", "PAY", "WITHDRAWAL", "-",
|
if debit_aliases and direction_val in debit_aliases:
|
||||||
"MƏXARIC", "MEXARIC", "ÇIXIB", "CIXIB", "XƏRC", "XERC"):
|
|
||||||
record["drcr"] = "D"
|
record["drcr"] = "D"
|
||||||
elif direction_val in ("C", "CR", "CREDIT", "CT", "IN", "RECEIVE", "DEPOSIT", "+",
|
elif credit_aliases and direction_val in credit_aliases:
|
||||||
"MƏDAXIL", "MEDAXIL", "DAXIL", "DAXIL OLUB", "GƏLIR", "GELIR"):
|
|
||||||
record["drcr"] = "C"
|
record["drcr"] = "C"
|
||||||
else:
|
else:
|
||||||
record["drcr"] = "D"
|
# Unknown direction value — surface to the user so they can add it
|
||||||
|
# to the preset, instead of silently guessing.
|
||||||
|
return None, "direction", direction_val
|
||||||
|
|
||||||
if not record["date"]:
|
if not record["date"]:
|
||||||
# Distinguish "row had a date input we couldn't parse" from "row had no date
|
# Distinguish "row had a date input we couldn't parse" from "row had no date
|
||||||
# input at all" — only the former is a user-actionable error.
|
# input at all" — only the former is a user-actionable error.
|
||||||
return None, "date" if saw_date_input else "amount"
|
return None, "date" if saw_date_input else "amount", None
|
||||||
if record["amount"] == 0:
|
if record["amount"] == 0:
|
||||||
return None, "amount"
|
return None, "amount", None
|
||||||
|
|
||||||
return record, ""
|
return record, "", None
|
||||||
|
|
||||||
|
|
||||||
def _parse_date(value, custom_format):
|
def _parse_date(value, custom_format):
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@ def parse_excel_for_preview(file_url, preset_name, bank_integration, bank_accoun
|
||||||
transactions = result["transactions"]
|
transactions = result["transactions"]
|
||||||
dropped_date = result.get("dropped_date", 0)
|
dropped_date = result.get("dropped_date", 0)
|
||||||
dropped_amount = result.get("dropped_amount", 0)
|
dropped_amount = result.get("dropped_amount", 0)
|
||||||
|
dropped_direction = result.get("dropped_direction", 0)
|
||||||
|
unknown_directions = result.get("unknown_directions", [])
|
||||||
|
|
||||||
if not transactions:
|
if not transactions:
|
||||||
return {
|
return {
|
||||||
|
|
@ -25,6 +27,8 @@ def parse_excel_for_preview(file_url, preset_name, bank_integration, bank_accoun
|
||||||
"total_parsed": 0,
|
"total_parsed": 0,
|
||||||
"dropped_date": dropped_date,
|
"dropped_date": dropped_date,
|
||||||
"dropped_amount": dropped_amount,
|
"dropped_amount": dropped_amount,
|
||||||
|
"dropped_direction": dropped_direction,
|
||||||
|
"unknown_directions": unknown_directions,
|
||||||
}
|
}
|
||||||
|
|
||||||
ref_nos = [t["ref_no"] for t in transactions if t.get("ref_no")]
|
ref_nos = [t["ref_no"] for t in transactions if t.get("ref_no")]
|
||||||
|
|
@ -56,6 +60,8 @@ def parse_excel_for_preview(file_url, preset_name, bank_integration, bank_accoun
|
||||||
"total_parsed": len(transactions),
|
"total_parsed": len(transactions),
|
||||||
"dropped_date": dropped_date,
|
"dropped_date": dropped_date,
|
||||||
"dropped_amount": dropped_amount,
|
"dropped_amount": dropped_amount,
|
||||||
|
"dropped_direction": dropped_direction,
|
||||||
|
"unknown_directions": unknown_directions,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -329,6 +335,8 @@ def load_registries_from_excel(file_url, preset_name, bank_integration,
|
||||||
transactions = result["transactions"]
|
transactions = result["transactions"]
|
||||||
dropped_date = result.get("dropped_date", 0)
|
dropped_date = result.get("dropped_date", 0)
|
||||||
dropped_amount = result.get("dropped_amount", 0)
|
dropped_amount = result.get("dropped_amount", 0)
|
||||||
|
dropped_direction = result.get("dropped_direction", 0)
|
||||||
|
unknown_directions = result.get("unknown_directions", [])
|
||||||
|
|
||||||
new_c = new_s = new_p = 0
|
new_c = new_s = new_p = 0
|
||||||
for txn in transactions:
|
for txn in transactions:
|
||||||
|
|
@ -359,4 +367,6 @@ def load_registries_from_excel(file_url, preset_name, bank_integration,
|
||||||
"new_purposes": new_p,
|
"new_purposes": new_p,
|
||||||
"dropped_date": dropped_date,
|
"dropped_date": dropped_date,
|
||||||
"dropped_amount": dropped_amount,
|
"dropped_amount": dropped_amount,
|
||||||
|
"dropped_direction": dropped_direction,
|
||||||
|
"unknown_directions": unknown_directions,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -227,6 +227,8 @@ function _show_load_data_dialog(frm) {
|
||||||
const res = r.message;
|
const res = r.message;
|
||||||
const droppedDate = res.dropped_date || 0;
|
const droppedDate = res.dropped_date || 0;
|
||||||
const droppedAmount = res.dropped_amount || 0;
|
const droppedAmount = res.dropped_amount || 0;
|
||||||
|
const droppedDirection = res.dropped_direction || 0;
|
||||||
|
const unknownDirections = res.unknown_directions || [];
|
||||||
let msg = __('From {0} rows — new customers: {1}, suppliers: {2}, purposes: {3}',
|
let msg = __('From {0} rows — new customers: {1}, suppliers: {2}, purposes: {3}',
|
||||||
[res.total_rows, res.new_customers, res.new_suppliers, res.new_purposes]);
|
[res.total_rows, res.new_customers, res.new_suppliers, res.new_purposes]);
|
||||||
if (droppedDate > 0) {
|
if (droppedDate > 0) {
|
||||||
|
|
@ -235,6 +237,13 @@ function _show_load_data_dialog(frm) {
|
||||||
[droppedDate]) +
|
[droppedDate]) +
|
||||||
'</span>';
|
'</span>';
|
||||||
}
|
}
|
||||||
|
if (droppedDirection > 0) {
|
||||||
|
const sample = unknownDirections.map(frappe.utils.escape_html).join(', ') || '?';
|
||||||
|
msg += '<br><br><span class="text-warning">' +
|
||||||
|
__('Also skipped {0} row(s) with unknown direction values: <code>{1}</code>. Add them to Debit/Credit Values on the preset.',
|
||||||
|
[droppedDirection, sample]) +
|
||||||
|
'</span>';
|
||||||
|
}
|
||||||
if (droppedAmount > 0) {
|
if (droppedAmount > 0) {
|
||||||
msg += '<br><span class="text-muted">' +
|
msg += '<br><span class="text-muted">' +
|
||||||
__('Also skipped {0} row(s) with missing/zero amount.', [droppedAmount]) +
|
__('Also skipped {0} row(s) with missing/zero amount.', [droppedAmount]) +
|
||||||
|
|
@ -242,7 +251,7 @@ function _show_load_data_dialog(frm) {
|
||||||
}
|
}
|
||||||
frappe.msgprint({
|
frappe.msgprint({
|
||||||
title: __('Data Loaded'),
|
title: __('Data Loaded'),
|
||||||
indicator: droppedDate > 0 ? 'orange' : 'green',
|
indicator: (droppedDate > 0 || droppedDirection > 0) ? 'orange' : 'green',
|
||||||
message: msg,
|
message: msg,
|
||||||
});
|
});
|
||||||
frm.reload_doc();
|
frm.reload_doc();
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@
|
||||||
"format_section",
|
"format_section",
|
||||||
"header_row",
|
"header_row",
|
||||||
"amount_mode",
|
"amount_mode",
|
||||||
|
"direction_debit_values",
|
||||||
|
"direction_credit_values",
|
||||||
"column_break_1",
|
"column_break_1",
|
||||||
"use_custom_date_format",
|
"use_custom_date_format",
|
||||||
"date_format",
|
"date_format",
|
||||||
|
|
@ -54,6 +56,20 @@
|
||||||
"label": "Amount Mode",
|
"label": "Amount Mode",
|
||||||
"options": "Single column with sign\nSeparate debit/credit columns\nSingle column + direction column"
|
"options": "Single column with sign\nSeparate debit/credit columns\nSingle column + direction column"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"depends_on": "eval:doc.amount_mode == 'Single column + direction column'",
|
||||||
|
"description": "Comma-separated values from the Direction column that mean DEBIT (outflow). Case-insensitive. Example: DR, Debit, Məxaric, -",
|
||||||
|
"fieldname": "direction_debit_values",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"label": "Debit Values"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"depends_on": "eval:doc.amount_mode == 'Single column + direction column'",
|
||||||
|
"description": "Comma-separated values from the Direction column that mean CREDIT (inflow). Case-insensitive. Example: CR, Credit, Mədaxil, +",
|
||||||
|
"fieldname": "direction_credit_values",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"label": "Credit Values"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"fieldname": "column_break_1",
|
"fieldname": "column_break_1",
|
||||||
"fieldtype": "Column Break"
|
"fieldtype": "Column Break"
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,16 @@
|
||||||
# Copyright (c) 2026, JeyERP and contributors
|
# Copyright (c) 2026, JeyERP and contributors
|
||||||
# For license information, please see license.txt
|
# For license information, please see license.txt
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
from frappe import _
|
||||||
from frappe.model.document import Document
|
from frappe.model.document import Document
|
||||||
|
|
||||||
|
|
||||||
class BankIntegrationExcelPreset(Document):
|
class BankIntegrationExcelPreset(Document):
|
||||||
pass
|
def validate(self):
|
||||||
|
if self.amount_mode == "Single column + direction column":
|
||||||
|
if not (self.direction_debit_values or "").strip() and not (self.direction_credit_values or "").strip():
|
||||||
|
frappe.throw(_(
|
||||||
|
"Mode 'Single column + direction column' requires at least one of "
|
||||||
|
"'Debit Values' or 'Credit Values' to be filled (comma-separated)."
|
||||||
|
))
|
||||||
|
|
|
||||||
|
|
@ -117,11 +117,17 @@ const BIExcelImport = {
|
||||||
const skipped = r.message.skipped_duplicates || 0;
|
const skipped = r.message.skipped_duplicates || 0;
|
||||||
const droppedDate = r.message.dropped_date || 0;
|
const droppedDate = r.message.dropped_date || 0;
|
||||||
const droppedAmount = r.message.dropped_amount || 0;
|
const droppedAmount = r.message.dropped_amount || 0;
|
||||||
|
const droppedDirection = r.message.dropped_direction || 0;
|
||||||
|
const unknownDirections = r.message.unknown_directions || [];
|
||||||
|
|
||||||
if (droppedDate > 0 && !txns.length) {
|
if (droppedDate > 0 && !txns.length) {
|
||||||
BIExcelImport._showDateParseHelp(droppedDate, values.preset);
|
BIExcelImport._showDateParseHelp(droppedDate, values.preset);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (droppedDirection > 0 && !txns.length) {
|
||||||
|
BIExcelImport._showDirectionHelp(droppedDirection, unknownDirections, values.preset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!txns.length) {
|
if (!txns.length) {
|
||||||
frappe.msgprint({
|
frappe.msgprint({
|
||||||
title: __('Nothing to Import'),
|
title: __('Nothing to Import'),
|
||||||
|
|
@ -132,12 +138,13 @@ const BIExcelImport = {
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
BIExcelImport._showPreview(txns, skipped, values, listview, droppedDate, droppedAmount);
|
BIExcelImport._showPreview(txns, skipped, values, listview,
|
||||||
|
droppedDate, droppedAmount, droppedDirection, unknownDirections);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
_showPreview(txns, skipped, values, listview, droppedDate, droppedAmount) {
|
_showPreview(txns, skipped, values, listview, droppedDate, droppedAmount, droppedDirection, unknownDirections) {
|
||||||
let table = '<div style="max-height: 500px; overflow-y: auto;">';
|
let table = '<div style="max-height: 500px; overflow-y: auto;">';
|
||||||
if (skipped > 0) {
|
if (skipped > 0) {
|
||||||
table += '<div class="alert alert-info">' +
|
table += '<div class="alert alert-info">' +
|
||||||
|
|
@ -150,6 +157,14 @@ const BIExcelImport = {
|
||||||
__("Open the Bank Integration Excel Preset, enable 'Use Custom Date Format', and set the exact format your bank uses.") +
|
__("Open the Bank Integration Excel Preset, enable 'Use Custom Date Format', and set the exact format your bank uses.") +
|
||||||
'</div>';
|
'</div>';
|
||||||
}
|
}
|
||||||
|
if (droppedDirection > 0) {
|
||||||
|
const sample = (unknownDirections || []).map(frappe.utils.escape_html).join(', ');
|
||||||
|
table += '<div class="alert alert-warning">' +
|
||||||
|
__('Dropped {0} row(s) — unknown direction value(s):', [droppedDirection]) +
|
||||||
|
' <code>' + (sample || '?') + '</code>. ' +
|
||||||
|
__('Open the preset and add these to Debit Values or Credit Values.') +
|
||||||
|
'</div>';
|
||||||
|
}
|
||||||
if (droppedAmount > 0) {
|
if (droppedAmount > 0) {
|
||||||
table += '<div class="alert alert-warning">' +
|
table += '<div class="alert alert-warning">' +
|
||||||
__('Dropped {0} row(s) — missing or zero amount.', [droppedAmount]) +
|
__('Dropped {0} row(s) — missing or zero amount.', [droppedAmount]) +
|
||||||
|
|
@ -301,6 +316,30 @@ const BIExcelImport = {
|
||||||
d.show();
|
d.show();
|
||||||
},
|
},
|
||||||
|
|
||||||
|
_showDirectionHelp(droppedCount, unknownDirections, presetName) {
|
||||||
|
const openPreset = function () {
|
||||||
|
frappe.set_route('Form', 'Bank Integration Excel Preset', presetName);
|
||||||
|
};
|
||||||
|
const sample = (unknownDirections || []).map(frappe.utils.escape_html).join(', ') || '?';
|
||||||
|
const d = new frappe.ui.Dialog({
|
||||||
|
title: __('Unknown Direction Values'),
|
||||||
|
fields: [{
|
||||||
|
fieldtype: 'HTML',
|
||||||
|
options:
|
||||||
|
'<div class="alert alert-danger">' +
|
||||||
|
__('All {0} row(s) were dropped because the Direction column contained values not configured on the preset.', [droppedCount]) +
|
||||||
|
'</div>' +
|
||||||
|
'<p>' + __('Unrecognised values:') + ' <code>' + sample + '</code></p>' +
|
||||||
|
'<p>' + __('Open the preset and add these to <b>Debit Values</b> (for outflows) or <b>Credit Values</b> (for inflows), comma-separated.') + '</p>',
|
||||||
|
}],
|
||||||
|
primary_action_label: __('Open Preset'),
|
||||||
|
primary_action() { d.hide(); openPreset(); },
|
||||||
|
secondary_action_label: __('Close'),
|
||||||
|
secondary_action() { d.hide(); },
|
||||||
|
});
|
||||||
|
d.show();
|
||||||
|
},
|
||||||
|
|
||||||
_closeProgress() {
|
_closeProgress() {
|
||||||
try {
|
try {
|
||||||
if (frappe.cur_progress) {
|
if (frappe.cur_progress) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue