feat(bank-integration): inline preview and auto-detect for File Format
Two new helpers on the Bank Integration form, shown when a Sample File is attached: - "Preview Sample" opens a modal with the header row + first 20 data rows, with badges on each column showing the currently mapped Standard Field (or "unmapped"). Helps spot wrong Header Row / column drift before importing real statements. - "Auto-detect Columns" reads the sample headers and replaces the Column Mappings table with a best-effort guess based on a multilingual synonym list (English, Azərbaycan, Russian). Reports how many of the total columns matched and lists unmapped headers so the user can fill the rest manually. Asks for confirmation before overwriting an existing table. Both helpers are backed by new whitelisted endpoints parse_sample_preview and auto_detect_column_mappings in excel_parser.py.
This commit is contained in:
parent
04b16a6eea
commit
05ee98cc0e
|
|
@ -382,6 +382,211 @@ def _parse_date(value, custom_format):
|
|||
return None
|
||||
|
||||
|
||||
# Heuristic synonyms for auto-detection. Lowercased; the matcher normalises
|
||||
# whitespace and Azərbaycan letters to Latin equivalents before comparing.
|
||||
# Order within each list does not matter — first hit wins for the column.
|
||||
# Order between standard fields matters: a header that could be both Purpose
|
||||
# and Description matches Purpose first (Purpose comes earlier in the dict).
|
||||
_HEADER_SYNONYMS = {
|
||||
"Date": (
|
||||
"date", "operation date", "transaction date", "value date", "posting date",
|
||||
"tarix", "tarıx", "əməliyyat tarixi", "tarix gun",
|
||||
"дата", "дата операции", "дата документа",
|
||||
),
|
||||
"Reference Number": (
|
||||
"reference", "reference number", "ref", "ref no", "ref number",
|
||||
"doc no", "document no", "document number", "transaction id", "txn id",
|
||||
"sənəd nömrəsi", "sənəd no", "sənəd", "ref nömrəsi",
|
||||
"номер документа", "номер", "док", "ссылка", "идентификатор",
|
||||
),
|
||||
"Counterparty": (
|
||||
"counterparty", "client name", "client", "customer name", "name", "partner",
|
||||
"müştəri", "qarşı tərəf", "tərəf-müqabil", "tərəf-muqabil",
|
||||
"контрагент", "наименование", "плательщик", "получатель",
|
||||
),
|
||||
"Counterparty Tax ID (VOEN)": (
|
||||
"voen", "tax id", "taxpayer id", "tin", "inn", "vat number",
|
||||
"müştəri voen", "qarşı tərəf voen",
|
||||
"инн", "налоговый номер",
|
||||
),
|
||||
"Counterparty IBAN": (
|
||||
"iban", "counterparty iban", "client iban", "client account",
|
||||
"müştəri iban", "müştəri hesab", "hesab nömrəsi",
|
||||
"счёт", "счет", "номер счёта", "номер счета",
|
||||
),
|
||||
"Amount": (
|
||||
"amount", "sum", "total", "value",
|
||||
"məbləğ",
|
||||
"сумма",
|
||||
),
|
||||
"Debit": (
|
||||
"debit", "withdrawal", "pay", "outflow", "debet",
|
||||
"məxaric", "ödəniş",
|
||||
"дебет", "расход", "списание",
|
||||
),
|
||||
"Credit": (
|
||||
"credit", "deposit", "receive", "inflow", "kredit",
|
||||
"mədaxil", "daxilolma",
|
||||
"кредит", "приход", "поступление", "зачисление",
|
||||
),
|
||||
"Direction": (
|
||||
"direction", "type", "drcr", "operation type", "movement",
|
||||
"məbləğ növü", "əməliyyat növü", "növ",
|
||||
"тип", "тип операции",
|
||||
),
|
||||
"Currency": (
|
||||
"currency", "ccy", "cur",
|
||||
"valyuta",
|
||||
"валюта",
|
||||
),
|
||||
"Purpose": (
|
||||
"purpose", "narrative", "details", "purpose of payment", "payment purpose",
|
||||
"təyinat", "ödənişin təyinatı", "ödəniş təyinatı",
|
||||
"назначение", "назначение платежа", "основание",
|
||||
),
|
||||
"Description": (
|
||||
"description", "comment", "comments", "remark", "remarks", "note", "notes", "memo",
|
||||
"qeyd", "şərh", "izahat",
|
||||
"комментарий", "примечание", "описание",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_AZERI_TO_LATIN = str.maketrans({
|
||||
"ə": "e", "Ə": "e", "ı": "i", "İ": "i", "ö": "o", "Ö": "o",
|
||||
"ü": "u", "Ü": "u", "ş": "s", "Ş": "s", "ç": "c", "Ç": "c",
|
||||
"ğ": "g", "Ğ": "g",
|
||||
})
|
||||
|
||||
|
||||
def _norm_for_match(s):
|
||||
if s is None:
|
||||
return ""
|
||||
s = str(s).strip().lower().translate(_AZERI_TO_LATIN)
|
||||
return " ".join(s.split())
|
||||
|
||||
|
||||
def _guess_standard_field(header):
|
||||
"""Return a Standard Field label for the given header text, or None."""
|
||||
h = _norm_for_match(header)
|
||||
if not h:
|
||||
return None
|
||||
# 1. Exact match first.
|
||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
if h == _norm_for_match(syn):
|
||||
return field
|
||||
# 2. Substring fallback: header *contains* a synonym (e.g. "Payment Reference No.").
|
||||
for field, synonyms in _HEADER_SYNONYMS.items():
|
||||
for syn in synonyms:
|
||||
syn_n = _norm_for_match(syn)
|
||||
if syn_n and len(syn_n) >= 4 and syn_n in h:
|
||||
return field
|
||||
return None
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def auto_detect_column_mappings(file_url, header_row=1):
|
||||
"""Read sample file headers and propose a mapping to Standard Fields.
|
||||
|
||||
Returns:
|
||||
{success: True, mappings: [{excel_column, standard_field, confidence}],
|
||||
unmatched: [...header strings...]}
|
||||
"""
|
||||
from frappe.utils import cint
|
||||
|
||||
header_row = max(cint(header_row) or 1, 1)
|
||||
if not file_url:
|
||||
return {"success": False, "message": "file_url is required"}
|
||||
|
||||
headers_result = parse_sample_headers(file_url, header_row=header_row)
|
||||
if not headers_result.get("success"):
|
||||
return headers_result
|
||||
|
||||
headers = headers_result.get("headers") or []
|
||||
mappings = []
|
||||
unmatched = []
|
||||
# Standard fields used so far — avoid mapping two columns to the same field.
|
||||
# (Debit + Credit are siblings, so we drop them from the "used" set after
|
||||
# the loop so both can be picked up.)
|
||||
used = set()
|
||||
for h in headers:
|
||||
guess = _guess_standard_field(h)
|
||||
if guess and guess not in used:
|
||||
mappings.append({"excel_column": h, "standard_field": guess})
|
||||
# Date / Reference / Amount / etc. should appear only once; Debit
|
||||
# and Credit are mutually exclusive halves of a row so they don't
|
||||
# collide with each other, but we still don't want two Debit columns.
|
||||
used.add(guess)
|
||||
else:
|
||||
mappings.append({"excel_column": h, "standard_field": ""})
|
||||
unmatched.append(h)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"mappings": mappings,
|
||||
"unmatched": unmatched,
|
||||
"total_headers": len(headers),
|
||||
"matched_count": sum(1 for m in mappings if m["standard_field"]),
|
||||
}
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def parse_sample_preview(file_url, header_row=1, max_rows=20):
|
||||
"""Return the header row + first N data rows of a sample file for preview UI."""
|
||||
from openpyxl import load_workbook
|
||||
from frappe.utils import cint
|
||||
|
||||
header_row = max(cint(header_row) or 1, 1)
|
||||
max_rows = max(min(cint(max_rows) or 20, 100), 1)
|
||||
|
||||
if not file_url:
|
||||
frappe.throw("file_url is required")
|
||||
|
||||
file_doc = frappe.get_doc("File", {"file_url": file_url})
|
||||
if file_doc.is_private and file_doc.owner != frappe.session.user:
|
||||
if not frappe.has_permission("File", "read", file_doc.name):
|
||||
frappe.throw("Not permitted to read this file")
|
||||
|
||||
file_path = file_doc.get_full_path()
|
||||
wb = load_workbook(file_path, data_only=True, read_only=True)
|
||||
try:
|
||||
ws = wb.active
|
||||
# Cap how much we pull: header_row + max_rows worth of cells.
|
||||
end_row = header_row + max_rows
|
||||
all_rows = list(ws.iter_rows(min_row=1, max_row=end_row, values_only=True))
|
||||
if not all_rows or header_row > len(all_rows):
|
||||
return {"success": True, "headers": [], "rows": [], "header_row": header_row}
|
||||
|
||||
raw_headers = all_rows[header_row - 1]
|
||||
headers = []
|
||||
col_indexes = []
|
||||
for i, cell in enumerate(raw_headers):
|
||||
if cell is None:
|
||||
continue
|
||||
h = str(cell).strip()
|
||||
if not h:
|
||||
continue
|
||||
headers.append(h)
|
||||
col_indexes.append(i)
|
||||
|
||||
rows = []
|
||||
for raw in all_rows[header_row:]:
|
||||
if not raw or all(c is None or str(c).strip() == "" for c in raw):
|
||||
continue
|
||||
out = []
|
||||
for i in col_indexes:
|
||||
cell = raw[i] if i < len(raw) else None
|
||||
out.append("" if cell is None else str(cell))
|
||||
rows.append(out)
|
||||
if len(rows) >= max_rows:
|
||||
break
|
||||
|
||||
return {"success": True, "headers": headers, "rows": rows, "header_row": header_row}
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def parse_sample_headers(file_url, header_row=1):
|
||||
"""Read just the header row of a sample file and return cleaned header strings.
|
||||
|
|
|
|||
|
|
@ -18,6 +18,18 @@ frappe.ui.form.on('Bank Integration', {
|
|||
refresh(frm) {
|
||||
BIFileFormat.applySampleHeaders(frm);
|
||||
|
||||
// === File Format helpers — available even on new records since they
|
||||
// help the user set up the import format from a sample file. ===
|
||||
if (frm.doc.sample_file) {
|
||||
frm.add_custom_button(__('Preview Sample'), () => {
|
||||
BIFileFormat.showPreview(frm);
|
||||
}, __('File Format'));
|
||||
|
||||
frm.add_custom_button(__('Auto-detect Columns'), () => {
|
||||
BIFileFormat.autoDetect(frm);
|
||||
}, __('File Format'));
|
||||
}
|
||||
|
||||
if (frm.is_new()) return;
|
||||
|
||||
// === Load Data (file-based, registry-only) ===
|
||||
|
|
@ -197,6 +209,154 @@ const BIFileFormat = {
|
|||
grid.update_docfield_property('excel_column', 'options', (headers || []).join('\n'));
|
||||
grid.refresh();
|
||||
},
|
||||
|
||||
showPreview(frm) {
|
||||
if (!frm.doc.sample_file) {
|
||||
frappe.msgprint(__('Upload a Sample Excel File first.'));
|
||||
return;
|
||||
}
|
||||
frappe.call({
|
||||
method: 'jey_erp.bank_integration.excel_parser.parse_sample_preview',
|
||||
args: {
|
||||
file_url: frm.doc.sample_file,
|
||||
header_row: frm.doc.header_row || 1,
|
||||
max_rows: 20,
|
||||
},
|
||||
callback: (r) => {
|
||||
if (!r.message || !r.message.success) {
|
||||
frappe.msgprint({
|
||||
title: __('Preview Error'),
|
||||
indicator: 'red',
|
||||
message: (r.message && r.message.message) || __('Could not read sample file.'),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const headers = r.message.headers || [];
|
||||
const rows = r.message.rows || [];
|
||||
const esc = frappe.utils.escape_html;
|
||||
|
||||
if (!headers.length) {
|
||||
frappe.msgprint({
|
||||
title: __('Empty Header Row'),
|
||||
indicator: 'orange',
|
||||
message: __("No headers found in row {0}. Adjust 'Header Row' and try again.", [frm.doc.header_row || 1]),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Build a header → standard_field map from the current grid so
|
||||
// the preview can show which columns are mapped.
|
||||
const mappedBy = {};
|
||||
(frm.doc.column_mappings || []).forEach(r => {
|
||||
if (r.excel_column && r.standard_field) {
|
||||
mappedBy[r.excel_column.trim().toLowerCase()] = r.standard_field;
|
||||
}
|
||||
});
|
||||
|
||||
let html = '<div style="max-height: 500px; overflow:auto;">';
|
||||
html += '<div class="text-muted small" style="margin-bottom:8px;">' +
|
||||
__('Showing first {0} data row(s) starting after Header Row {1}.',
|
||||
[rows.length, r.message.header_row]) + '</div>';
|
||||
html += '<table class="table table-bordered" style="font-size:12px; white-space:nowrap;">';
|
||||
html += '<thead style="position:sticky; top:0; background:var(--bg-color); z-index:1;">';
|
||||
html += '<tr>';
|
||||
headers.forEach(h => {
|
||||
const std = mappedBy[h.trim().toLowerCase()];
|
||||
const badge = std
|
||||
? ` <span class="label label-success" style="font-size:9px;">${esc(std)}</span>`
|
||||
: ` <span class="label label-default" style="font-size:9px;">${__('unmapped')}</span>`;
|
||||
html += `<th>${esc(h)}${badge}</th>`;
|
||||
});
|
||||
html += '</tr></thead><tbody>';
|
||||
rows.forEach(row => {
|
||||
html += '<tr>';
|
||||
row.forEach(cell => {
|
||||
html += `<td>${esc(cell)}</td>`;
|
||||
});
|
||||
html += '</tr>';
|
||||
});
|
||||
html += '</tbody></table></div>';
|
||||
|
||||
const d = new frappe.ui.Dialog({
|
||||
title: __('Sample File Preview'),
|
||||
size: 'extra-large',
|
||||
fields: [{ fieldname: 'preview_html', fieldtype: 'HTML', options: html }],
|
||||
primary_action_label: __('Close'),
|
||||
primary_action() { d.hide(); },
|
||||
});
|
||||
d.$wrapper.find('.modal-dialog').css({ 'max-width': '95%', 'width': '95%' });
|
||||
d.show();
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
autoDetect(frm) {
|
||||
if (!frm.doc.sample_file) {
|
||||
frappe.msgprint(__('Upload a Sample Excel File first.'));
|
||||
return;
|
||||
}
|
||||
const existing = (frm.doc.column_mappings || []).filter(r => r.excel_column || r.standard_field);
|
||||
const proceed = () => {
|
||||
frappe.call({
|
||||
method: 'jey_erp.bank_integration.excel_parser.auto_detect_column_mappings',
|
||||
args: {
|
||||
file_url: frm.doc.sample_file,
|
||||
header_row: frm.doc.header_row || 1,
|
||||
},
|
||||
callback: (r) => {
|
||||
if (!r.message || !r.message.success) {
|
||||
frappe.msgprint({
|
||||
title: __('Auto-detect Error'),
|
||||
indicator: 'red',
|
||||
message: (r.message && r.message.message) || __('Could not read sample file.'),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const mappings = r.message.mappings || [];
|
||||
if (!mappings.length) {
|
||||
frappe.msgprint({
|
||||
title: __('No Headers Found'),
|
||||
indicator: 'orange',
|
||||
message: __("No headers found in row {0}. Adjust 'Header Row'.", [frm.doc.header_row || 1]),
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Replace the table contents in-place.
|
||||
frm.clear_table('column_mappings');
|
||||
mappings.forEach(m => {
|
||||
const row = frm.add_child('column_mappings');
|
||||
row.excel_column = m.excel_column;
|
||||
row.standard_field = m.standard_field || '';
|
||||
});
|
||||
frm.refresh_field('column_mappings');
|
||||
frm.dirty();
|
||||
|
||||
const matched = r.message.matched_count || 0;
|
||||
const total = r.message.total_headers || mappings.length;
|
||||
const unmatched = r.message.unmatched || [];
|
||||
let msg = __('Mapped {0} of {1} column(s).', [matched, total]);
|
||||
if (unmatched.length) {
|
||||
msg += '<br><br>' + __('Unmapped columns (set Standard Field manually if needed):') +
|
||||
'<br><code>' + unmatched.map(frappe.utils.escape_html).join(', ') + '</code>';
|
||||
}
|
||||
msg += '<br><br>' + __('Review the table, then <b>Save</b> the form to confirm.');
|
||||
frappe.msgprint({
|
||||
title: __('Auto-detect Complete'),
|
||||
indicator: matched === total ? 'green' : (matched > 0 ? 'orange' : 'red'),
|
||||
message: msg,
|
||||
});
|
||||
},
|
||||
});
|
||||
};
|
||||
if (existing.length) {
|
||||
frappe.confirm(
|
||||
__('This will replace the {0} existing Column Mapping row(s). Continue?', [existing.length]),
|
||||
proceed,
|
||||
);
|
||||
} else {
|
||||
proceed();
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
function _safe(frm, action) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue