feat(bank-integration): redesign Excel Preset UX with Detect & preview

Standard Field is now a Link to a new "Bank Integration Standard Field"
doctype (autoname=Prompt). Each entry holds a display name like "Date" /
"Reference Number" — the field that the user picks in Column Mappings —
and a `field_name` column with the technical key used by the parser.
Twelve standard fields ship as a fixture. The change replaces the old
Select with technical names (date, reference_number, debit, ...) which
were noisy in the UI.

Excel Preset gets two new fields:
  - sample_file (Attach) — user uploads a sample of their bank's xlsx
  - preview_html (HTML) — first 5 rows are rendered after detection

A "Detect Columns from Sample" custom button on the form calls a new
whitelisted endpoint detect_columns_from_sample() that reads the header
row + first 5 data rows. The Column Mappings table is repopulated with
one row per Excel header (Standard Field left empty for the user to
fill in), and the preview HTML shows what the parser actually sees so
the user can validate header_row / skip_rows. After detection the
sample File doc is removed via delete_sample_file().

excel_parser now resolves row.standard_field through Standard Field
to obtain the technical field_name. The Generic Bank Statement fixture
is updated to reference the new Link values.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

jey_erp/bank_integration/excel_parser.py

@@ -17,10 +17,17 @@ def parse_excel(file_url, preset_name):
 
 	preset = frappe.get_doc("Bank Integration Excel Preset", preset_name)
 
+	# row.standard_field is a Link to Bank Integration Standard Field whose
+	# `field_name` holds the technical key (date / reference_number / debit / ...).
 	col_to_standard = {}
 	for row in preset.column_mappings:
-		if row.excel_column and row.standard_field:
-			col_to_standard[row.excel_column.strip()] = row.standard_field
+		if not row.excel_column or not row.standard_field:
+			continue
+		field_name = frappe.db.get_value(
+			"Bank Integration Standard Field", row.standard_field, "field_name"
+		)
+		if field_name:
+			col_to_standard[row.excel_column.strip()] = field_name
 
 	if not col_to_standard:
 		frappe.throw(f"Preset '{preset_name}' has no column mappings")

jey_erp/bank_integration/import_api.py

@@ -244,6 +244,81 @@ def _upsert_purpose(purpose, drcr, bank_integration):
 		return False
 
 
+@frappe.whitelist()
+def detect_columns_from_sample(file_url, header_row=1):
+	"""Read column headers + first 5 data rows from a sample Excel file.
+
+	Returns:
+	    {success, headers: [...], preview_rows: [[...], ...]}
+	The frontend uses this to populate column_mappings and the preview HTML.
+	After detection the frontend deletes the sample File doc.
+	"""
+	from openpyxl import load_workbook
+
+	header_row = int(header_row or 1)
+
+	if not file_url:
+		frappe.throw(_("Sample file is required"))
+
+	file_doc = frappe.get_doc("File", {"file_url": file_url})
+	file_path = file_doc.get_full_path()
+
+	wb = load_workbook(file_path, data_only=True, read_only=True)
+	try:
+		ws = wb.active
+		all_rows = list(ws.iter_rows(values_only=True))
+
+		header_idx = header_row - 1
+		if header_idx < 0 or header_idx >= len(all_rows):
+			return {"success": False, "message": _("Header row {0} is out of range (file has {1} rows)").format(header_row, len(all_rows))}
+
+		headers_raw = all_rows[header_idx]
+		headers = [str(c).strip() if c is not None else "" for c in headers_raw]
+
+		# Trim trailing empty header columns
+		while headers and not headers[-1]:
+			headers.pop()
+
+		if not headers:
+			return {"success": False, "message": _("No column headers found in row {0}").format(header_row)}
+
+		preview_rows = []
+		for raw in all_rows[header_idx + 1: header_idx + 6]:
+			row_cells = []
+			for i in range(len(headers)):
+				val = raw[i] if i < len(raw) else None
+				if val is None:
+					row_cells.append("")
+				elif hasattr(val, "isoformat"):
+					row_cells.append(val.isoformat())
+				else:
+					row_cells.append(str(val))
+			preview_rows.append(row_cells)
+
+		return {
+			"success": True,
+			"headers": headers,
+			"preview_rows": preview_rows,
+		}
+	finally:
+		wb.close()
+
+
+@frappe.whitelist()
+def delete_sample_file(file_url):
+	"""Remove a Sample Excel File after detection. Idempotent."""
+	if not file_url:
+		return {"success": True}
+	try:
+		name = frappe.db.get_value("File", {"file_url": file_url}, "name")
+		if name:
+			frappe.delete_doc("File", name, ignore_permissions=True, force=True)
+			frappe.db.commit()
+	except Exception as e:
+		frappe.log_error(f"delete_sample_file failed for {file_url}: {e}", "BI Sample Cleanup")
+	return {"success": True}
+
+
 @frappe.whitelist()
 def update_bank_account_bi_default(bank_account, bi_type, bi_name):
 	"""Save the user's BRT-time choice to the BA's hidden fields."""

jey_erp/fixtures/bank_integration_excel_preset.json

@@ -9,56 +9,16 @@
   "date_format": "%Y-%m-%d",
   "amount_mode": "Separate debit/credit columns",
   "column_mappings": [
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Date",
-    "standard_field": "date"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Reference",
-    "standard_field": "reference_number"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Counterparty",
-    "standard_field": "counterparty"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Tax ID",
-    "standard_field": "counterparty_tax_id"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "IBAN",
-    "standard_field": "counterparty_iban"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Debit",
-    "standard_field": "debit"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Credit",
-    "standard_field": "credit"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Currency",
-    "standard_field": "currency"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Purpose",
-    "standard_field": "purpose"
-   },
-   {
-    "doctype": "Bank Integration Excel Column Mapping",
-    "excel_column": "Description",
-    "standard_field": "description"
-   }
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Date",         "standard_field": "Date"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Reference",    "standard_field": "Reference Number"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Counterparty", "standard_field": "Counterparty"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Tax ID",       "standard_field": "Counterparty Tax ID"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "IBAN",         "standard_field": "Counterparty IBAN"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Debit",        "standard_field": "Debit"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Credit",       "standard_field": "Credit"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Currency",     "standard_field": "Currency"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Purpose",      "standard_field": "Purpose"},
+   {"doctype": "Bank Integration Excel Column Mapping", "excel_column": "Description",  "standard_field": "Description"}
   ]
  }
 ]

jey_erp/fixtures/bank_integration_standard_field.json

@@ -0,0 +1,14 @@
+[
+ {"doctype": "Bank Integration Standard Field", "name": "Date",                "field_name": "date",                "description": "Transaction date (parsed using preset's date_format)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Reference Number",    "field_name": "reference_number",    "description": "Unique reference / document number from the bank statement."},
+ {"doctype": "Bank Integration Standard Field", "name": "Counterparty",        "field_name": "counterparty",        "description": "Sender or receiver name."},
+ {"doctype": "Bank Integration Standard Field", "name": "Counterparty Tax ID", "field_name": "counterparty_tax_id", "description": "Counterparty's VOEN / TIN / INN."},
+ {"doctype": "Bank Integration Standard Field", "name": "Counterparty IBAN",   "field_name": "counterparty_iban",   "description": "Counterparty's bank account / IBAN."},
+ {"doctype": "Bank Integration Standard Field", "name": "Amount",              "field_name": "amount",              "description": "Transaction amount (use with 'Single column with sign' or 'Single column + direction column' modes)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Debit",               "field_name": "debit",               "description": "Debit / outgoing amount column (use with 'Separate debit/credit columns' mode)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Credit",              "field_name": "credit",              "description": "Credit / incoming amount column (use with 'Separate debit/credit columns' mode)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Direction",           "field_name": "direction",           "description": "D/C / Debit/Credit indicator (use with 'Single column + direction column' mode)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Currency",            "field_name": "currency",            "description": "Currency code (AZN, USD, EUR, etc.)."},
+ {"doctype": "Bank Integration Standard Field", "name": "Purpose",             "field_name": "purpose",             "description": "Payment purpose / narrative."},
+ {"doctype": "Bank Integration Standard Field", "name": "Description",         "field_name": "description",         "description": "Additional info / notes."}
+]

jey_erp/hooks.py

@@ -158,6 +158,9 @@ fixtures = [
     {
         "doctype": "Print Format"
     },
+    {
+        "doctype": "Bank Integration Standard Field"
+    },
     {
         "doctype": "Bank Integration Excel Preset"
     },

jey_erp/jey_erp/doctype/bank_integration_excel_column_mapping/bank_integration_excel_column_mapping.json

@@ -19,10 +19,10 @@
   },
   {
    "fieldname": "standard_field",
-   "fieldtype": "Select",
+   "fieldtype": "Link",
    "in_list_view": 1,
    "label": "Standard Field",
-   "options": "\ndate\nreference_number\ncounterparty\ncounterparty_tax_id\ncounterparty_iban\namount\ndebit\ncredit\ndirection\ncurrency\npurpose\ndescription",
+   "options": "Bank Integration Standard Field",
    "reqd": 1
   },
   {

jey_erp/jey_erp/doctype/bank_integration_excel_preset/bank_integration_excel_preset.js

@@ -0,0 +1,135 @@
+// Bank Integration Excel Preset form: "Detect Columns from Sample" button.
+// Workflow:
+//   1. User uploads a sample of their bank's xlsx into Sample File.
+//   2. Clicks "Detect Columns from Sample" → backend reads the header row
+//      and first 5 data rows.
+//   3. Column Mappings table is repopulated: one row per Excel header,
+//      Standard Field left empty for the user to fill in.
+//   4. Preview HTML below shows the first 5 rows so the user can check
+//      they picked the right header row / skip rows.
+//   5. Sample File is deleted; field is cleared.
+
+frappe.ui.form.on("Bank Integration Excel Preset", {
+	refresh(frm) {
+		frm.add_custom_button(__("Detect Columns from Sample"), () => {
+			BIPresetDetect.run(frm);
+		});
+
+		// Render existing preview content if any (kept in memory, not persisted)
+		if (frm._bi_preview_html) {
+			BIPresetDetect.renderPreview(frm, frm._bi_preview_html);
+		}
+	},
+
+	sample_file(frm) {
+		// User changed sample file — clear stale preview
+		if (!frm.doc.sample_file) {
+			frm._bi_preview_html = "";
+			BIPresetDetect.renderPreview(frm, "");
+		}
+	},
+});
+
+const BIPresetDetect = {
+	run(frm) {
+		if (!frm.doc.sample_file) {
+			frappe.msgprint({
+				title: __("Sample File Required"),
+				indicator: "orange",
+				message: __("Upload a sample Excel file first (the 'Sample Excel File' field)."),
+			});
+			return;
+		}
+
+		const headerRow = frm.doc.header_row || 1;
+		frappe.show_alert({ message: __("Detecting columns..."), indicator: "blue" });
+
+		frappe.call({
+			method: "jey_erp.bank_integration.import_api.detect_columns_from_sample",
+			args: {
+				file_url: frm.doc.sample_file,
+				header_row: headerRow,
+			},
+			callback(r) {
+				if (!r.message || !r.message.success) {
+					frappe.msgprint({
+						title: __("Detection Failed"),
+						indicator: "red",
+						message: (r.message && r.message.message) || __("Could not parse sample file."),
+					});
+					return;
+				}
+
+				const headers = r.message.headers || [];
+				const previewRows = r.message.preview_rows || [];
+
+				BIPresetDetect.populateMappings(frm, headers);
+				BIPresetDetect.buildAndShowPreview(frm, headers, previewRows);
+				BIPresetDetect.cleanupSample(frm);
+
+				frappe.show_alert({
+					message: __("Detected {0} column(s). Pick a Standard Field for each.", [headers.length]),
+					indicator: "green",
+				}, 6);
+			},
+		});
+	},
+
+	populateMappings(frm, headers) {
+		// Replace existing column_mappings rows with new ones (one per header).
+		frm.clear_table("column_mappings");
+		headers.forEach((h) => {
+			if (!h) return;
+			const row = frm.add_child("column_mappings");
+			row.excel_column = h;
+			row.standard_field = "";
+		});
+		frm.refresh_field("column_mappings");
+	},
+
+	buildAndShowPreview(frm, headers, rows) {
+		let html = '<div style="max-height:300px; overflow:auto;">';
+		html += '<table class="table table-bordered table-condensed" style="font-size:0.85em;">';
+		html += '<thead><tr>';
+		headers.forEach((h) => {
+			html += '<th style="white-space:nowrap;">' + frappe.utils.escape_html(h || "") + '</th>';
+		});
+		html += '</tr></thead><tbody>';
+		rows.forEach((r) => {
+			html += '<tr>';
+			for (let i = 0; i < headers.length; i++) {
+				const cell = (i < r.length ? r[i] : "");
+				html += '<td>' + frappe.utils.escape_html(cell || "") + '</td>';
+			}
+			html += '</tr>';
+		});
+		html += '</tbody></table></div>';
+
+		frm._bi_preview_html = html;
+		BIPresetDetect.renderPreview(frm, html);
+	},
+
+	renderPreview(frm, html) {
+		const field = frm.get_field("preview_html");
+		if (!field) return;
+		field.html(html || '<div class="text-muted">' + __("No preview yet — upload a sample and click Detect.") + '</div>');
+	},
+
+	cleanupSample(frm) {
+		const fileUrl = frm.doc.sample_file;
+		if (!fileUrl) return;
+
+		// Clear field locally; if the doc is already saved, also persist
+		frm.set_value("sample_file", "");
+
+		frappe.call({
+			method: "jey_erp.bank_integration.import_api.delete_sample_file",
+			args: { file_url: fileUrl },
+		});
+
+		// If preset is already saved, persist the cleared field
+		if (!frm.is_new()) {
+			frm.save();
+		}
+	},
+};

jey_erp/jey_erp/doctype/bank_integration_excel_preset/bank_integration_excel_preset.json

@@ -13,8 +13,13 @@
   "date_format",
   "column_break_1",
   "amount_mode",
+  "sample_section",
+  "sample_file",
+  "sample_help_html",
   "columns_section",
-  "column_mappings"
+  "column_mappings",
+  "preview_section",
+  "preview_html"
  ],
  "fields": [
   {
@@ -70,6 +75,23 @@
    "label": "Amount Mode",
    "options": "Single column with sign\nSeparate debit/credit columns\nSingle column + direction column"
   },
+  {
+   "fieldname": "sample_section",
+   "fieldtype": "Section Break",
+   "label": "Sample File (auto-detect helper)",
+   "collapsible": 1
+  },
+  {
+   "fieldname": "sample_file",
+   "fieldtype": "Attach",
+   "label": "Sample Excel File",
+   "description": "Upload a sample of your bank's Excel export. Click 'Detect Columns from Sample' to populate Column Mappings and preview the first 5 parsed rows. The file is removed after detection."
+  },
+  {
+   "fieldname": "sample_help_html",
+   "fieldtype": "HTML",
+   "options": "<div class=\"text-muted\" style=\"margin-top:6px;\">After upload, click <b>Detect Columns from Sample</b> in the top-right menu. The button reads the header row, fills the table below with empty Standard Field rows, and shows a 5-row preview at the bottom. Then pick a Standard Field for each column.</div>"
+  },
   {
    "fieldname": "columns_section",
    "fieldtype": "Section Break",
@@ -80,6 +102,16 @@
    "fieldtype": "Table",
    "label": "Column Mappings",
    "options": "Bank Integration Excel Column Mapping"
+  },
+  {
+   "fieldname": "preview_section",
+   "fieldtype": "Section Break",
+   "label": "Sample Preview"
+  },
+  {
+   "fieldname": "preview_html",
+   "fieldtype": "HTML",
+   "label": "Preview"
   }
  ],
  "index_web_pages_for_search": 1,

jey_erp/jey_erp/doctype/bank_integration_standard_field/bank_integration_standard_field.json

@@ -0,0 +1,62 @@
+{
+ "actions": [],
+ "autoname": "Prompt",
+ "creation": "2026-05-08 00:00:00.000000",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "field_name",
+  "description"
+ ],
+ "fields": [
+  {
+   "fieldname": "field_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Internal Field Name",
+   "reqd": 1,
+   "unique": 1,
+   "description": "Technical key used by the parser (date, reference_number, debit, etc.)"
+  },
+  {
+   "fieldname": "description",
+   "fieldtype": "Small Text",
+   "label": "Description"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-05-08 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Jey Erp",
+ "name": "Bank Integration Standard Field",
+ "naming_rule": "By script",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Accounts User",
+   "share": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "ASC",
+ "states": [],
+ "track_changes": 0
+}

jey_erp/jey_erp/doctype/bank_integration_standard_field/bank_integration_standard_field.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2026, JeyERP and contributors
+# For license information, please see license.txt
+
+from frappe.model.document import Document
+
+
+class BankIntegrationStandardField(Document):
+	pass