ali
/
taxes_az
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(master-data): drop file-hash fast-path so sync always reconciles DB 

The hash fast-path compared the JSON file hash to a stored value and exited
early when unchanged. But it only tracked the *file*, not the *table*: if rows
went missing from the DB (manual delete, restore from an older dump) while the
JSON stayed the same, the sync skipped entirely and never restored them — so
`bench migrate` silently failed to recreate missing master-data rows.

Always run the full diff now. It costs one SELECT plus an in-memory compare —
measured at ~0.1s for 13k rows — so there's no reason to skip it. The diff is
driven by DB state, making the sync self-healing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ali 2026-05-20 16:14:50 +00:00
parent 34adb643ee
commit b963422bca
1 changed files with 18 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
taxes_az/master_data/sync.py
View File

@ -1,16 +1,20 @@
"""Hash-diff sync for master data shipped as JSON. """Diff-based sync for master data shipped as JSON.
The legacy approach (fixtures hook) re-imports every record on every `bench The legacy approach (fixtures hook) re-imports every record on every `bench
migrate` via DELETE+INSERT. For static reference data with thousands of rows migrate` via DELETE+INSERT. For static reference data with thousands of rows
that's a multi-minute hit. This module compares the file hash first (fast-path that's a multi-minute hit. This module instead reconciles the DB against the
exit when unchanged) and otherwise applies only the diff: insert new rows, JSON on every migrate: it reads the current rows in one query and applies only
update changed rows, delete rows missing from the file (or disable / skip when the delta — insert new rows, update changed rows, delete rows missing from the
they are referenced by existing documents). file (or disable / skip when they are referenced by existing documents).
The diff is intentionally driven by the *DB state*, not a stored file hash:
if rows go missing from the table (manual delete, restore from an old dump),
the next migrate restores them. A full diff costs one SELECT plus an in-memory
comparison — sub-second even for ~13k rows.
""" """
from __future__ import annotations from __future__ import annotations
import hashlib
import json import json
import os import os
from collections.abc import Iterable from collections.abc import Iterable
@ -18,18 +22,6 @@ from collections.abc import Iterable
import frappe import frappe
def _file_hash(path: str) -> str:
h = hashlib.md5(usedforsecurity=False)
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(8192), b""):
h.update(chunk)
return h.hexdigest()
def _global_key(doctype: str) -> str:
return f"taxes_az:master_data_hash:{frappe.scrub(doctype)}"
def _load_records(path: str, key_field: str) -> dict[str, dict]: def _load_records(path: str, key_field: str) -> dict[str, dict]:
with open(path, encoding="utf-8") as f: with open(path, encoding="utf-8") as f:
raw = json.load(f) raw = json.load(f)
@ -86,10 +78,10 @@ def sync_master_data(
is_tree: bool = False, is_tree: bool = False,
manage_deletes: bool = True, manage_deletes: bool = True,
) -> dict: ) -> dict:
"""Sync a doctype from a JSON file via per-record hash-diff. """Reconcile a doctype against a JSON file, applying only the delta.
Fast-path: if the file hash matches what was stored on the previous run, Runs a full diff against the current DB state every call — no file-hash
return immediately without touching the DB. fast-path, so rows that go missing from the table are always restored.
is_tree: doctype is a nested set — insert via db_insert() and rebuild is_tree: doctype is a nested set — insert via db_insert() and rebuild
lft/rgt once at the end (avoids per-row tree recomputation and the lft/rgt once at the end (avoids per-row tree recomputation and the
@ -102,11 +94,6 @@ def sync_master_data(
frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}") frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}")
return {"skipped": True} return {"skipped": True}
current_hash = _file_hash(json_path)
stored_hash = frappe.db.get_global(_global_key(doctype))
if stored_hash == current_hash:
return {"unchanged": True}
target = _load_records(json_path, key_field) target = _load_records(json_path, key_field)
if not target: if not target:
return {"empty_file": True} return {"empty_file": True}
@ -195,7 +182,6 @@ def sync_master_data(
rebuild_tree(doctype) rebuild_tree(doctype)
frappe.db.set_global(_global_key(doctype), current_hash)
frappe.db.commit() frappe.db.commit()
summary = { summary = {
@ -206,10 +192,11 @@ def sync_master_data(
"disabled": disabled, "disabled": disabled,
"skipped": skipped, "skipped": skipped,
} }
print( if inserted or updated or deleted or disabled or skipped:
f"[taxes_az] {doctype}: " print(
f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})" f"[taxes_az] {doctype}: "
) f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
)
return summary return summary