From b963422bca4dd99ea3dd0d564095185ee0eb049e Mon Sep 17 00:00:00 2001
From: Ali <010109ali@gmail.com>
Date: Wed, 20 May 2026 16:14:50 +0000
Subject: [PATCH] fix(master-data): drop file-hash fast-path so sync always
 reconciles DB
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The hash fast-path compared the JSON file hash to a stored value and exited
early when unchanged. But it only tracked the *file*, not the *table*: if rows
went missing from the DB (manual delete, restore from an older dump) while the
JSON stayed the same, the sync skipped entirely and never restored them — so
`bench migrate` silently failed to recreate missing master-data rows.

Always run the full diff now. It costs one SELECT plus an in-memory compare —
measured at ~0.1s for 13k rows — so there's no reason to skip it. The diff is
driven by DB state, making the sync self-healing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 taxes_az/master_data/sync.py | 49 +++++++++++++-----------------------
 1 file changed, 18 insertions(+), 31 deletions(-)

diff --git a/taxes_az/master_data/sync.py b/taxes_az/master_data/sync.py
index 00b1499..2ee8c02 100644
--- a/taxes_az/master_data/sync.py
+++ b/taxes_az/master_data/sync.py
@@ -1,16 +1,20 @@
-"""Hash-diff sync for master data shipped as JSON.
+"""Diff-based sync for master data shipped as JSON.
 
 The legacy approach (fixtures hook) re-imports every record on every `bench
 migrate` via DELETE+INSERT. For static reference data with thousands of rows
-that's a multi-minute hit. This module compares the file hash first (fast-path
-exit when unchanged) and otherwise applies only the diff: insert new rows,
-update changed rows, delete rows missing from the file (or disable / skip when
-they are referenced by existing documents).
+that's a multi-minute hit. This module instead reconciles the DB against the
+JSON on every migrate: it reads the current rows in one query and applies only
+the delta — insert new rows, update changed rows, delete rows missing from the
+file (or disable / skip when they are referenced by existing documents).
+
+The diff is intentionally driven by the *DB state*, not a stored file hash:
+if rows go missing from the table (manual delete, restore from an old dump),
+the next migrate restores them. A full diff costs one SELECT plus an in-memory
+comparison — sub-second even for ~13k rows.
 """
 
 from __future__ import annotations
 
-import hashlib
 import json
 import os
 from collections.abc import Iterable
@@ -18,18 +22,6 @@ from collections.abc import Iterable
 import frappe
 
 
-def _file_hash(path: str) -> str:
-    h = hashlib.md5(usedforsecurity=False)
-    with open(path, "rb") as f:
-        for chunk in iter(lambda: f.read(8192), b""):
-            h.update(chunk)
-    return h.hexdigest()
-
-
-def _global_key(doctype: str) -> str:
-    return f"taxes_az:master_data_hash:{frappe.scrub(doctype)}"
-
-
 def _load_records(path: str, key_field: str) -> dict[str, dict]:
     with open(path, encoding="utf-8") as f:
         raw = json.load(f)
@@ -86,10 +78,10 @@ def sync_master_data(
     is_tree: bool = False,
     manage_deletes: bool = True,
 ) -> dict:
-    """Sync a doctype from a JSON file via per-record hash-diff.
+    """Reconcile a doctype against a JSON file, applying only the delta.
 
-    Fast-path: if the file hash matches what was stored on the previous run,
-    return immediately without touching the DB.
+    Runs a full diff against the current DB state every call — no file-hash
+    fast-path, so rows that go missing from the table are always restored.
 
     is_tree: doctype is a nested set — insert via db_insert() and rebuild
         lft/rgt once at the end (avoids per-row tree recomputation and the
@@ -102,11 +94,6 @@ def sync_master_data(
         frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}")
         return {"skipped": True}
 
-    current_hash = _file_hash(json_path)
-    stored_hash = frappe.db.get_global(_global_key(doctype))
-    if stored_hash == current_hash:
-        return {"unchanged": True}
-
     target = _load_records(json_path, key_field)
     if not target:
         return {"empty_file": True}
@@ -195,7 +182,6 @@ def sync_master_data(
 
         rebuild_tree(doctype)
 
-    frappe.db.set_global(_global_key(doctype), current_hash)
     frappe.db.commit()
 
     summary = {
@@ -206,10 +192,11 @@ def sync_master_data(
         "disabled": disabled,
         "skipped": skipped,
     }
-    print(
-        f"[taxes_az] {doctype}: "
-        f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
-    )
+    if inserted or updated or deleted or disabled or skipped:
+        print(
+            f"[taxes_az] {doctype}: "
+            f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
+        )
     return summary