fix(master-data): drop file-hash fast-path so sync always reconciles DB
The hash fast-path compared the JSON file hash to a stored value and exited early when unchanged. But it only tracked the *file*, not the *table*: if rows went missing from the DB (manual delete, restore from an older dump) while the JSON stayed the same, the sync skipped entirely and never restored them — so `bench migrate` silently failed to recreate missing master-data rows. Always run the full diff now. It costs one SELECT plus an in-memory compare — measured at ~0.1s for 13k rows — so there's no reason to skip it. The diff is driven by DB state, making the sync self-healing. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
34adb643ee
commit
b963422bca
|
|
@ -1,16 +1,20 @@
|
|||
"""Hash-diff sync for master data shipped as JSON.
|
||||
"""Diff-based sync for master data shipped as JSON.
|
||||
|
||||
The legacy approach (fixtures hook) re-imports every record on every `bench
|
||||
migrate` via DELETE+INSERT. For static reference data with thousands of rows
|
||||
that's a multi-minute hit. This module compares the file hash first (fast-path
|
||||
exit when unchanged) and otherwise applies only the diff: insert new rows,
|
||||
update changed rows, delete rows missing from the file (or disable / skip when
|
||||
they are referenced by existing documents).
|
||||
that's a multi-minute hit. This module instead reconciles the DB against the
|
||||
JSON on every migrate: it reads the current rows in one query and applies only
|
||||
the delta — insert new rows, update changed rows, delete rows missing from the
|
||||
file (or disable / skip when they are referenced by existing documents).
|
||||
|
||||
The diff is intentionally driven by the *DB state*, not a stored file hash:
|
||||
if rows go missing from the table (manual delete, restore from an old dump),
|
||||
the next migrate restores them. A full diff costs one SELECT plus an in-memory
|
||||
comparison — sub-second even for ~13k rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
|
|
@ -18,18 +22,6 @@ from collections.abc import Iterable
|
|||
import frappe
|
||||
|
||||
|
||||
def _file_hash(path: str) -> str:
|
||||
h = hashlib.md5(usedforsecurity=False)
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _global_key(doctype: str) -> str:
|
||||
return f"taxes_az:master_data_hash:{frappe.scrub(doctype)}"
|
||||
|
||||
|
||||
def _load_records(path: str, key_field: str) -> dict[str, dict]:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
raw = json.load(f)
|
||||
|
|
@ -86,10 +78,10 @@ def sync_master_data(
|
|||
is_tree: bool = False,
|
||||
manage_deletes: bool = True,
|
||||
) -> dict:
|
||||
"""Sync a doctype from a JSON file via per-record hash-diff.
|
||||
"""Reconcile a doctype against a JSON file, applying only the delta.
|
||||
|
||||
Fast-path: if the file hash matches what was stored on the previous run,
|
||||
return immediately without touching the DB.
|
||||
Runs a full diff against the current DB state every call — no file-hash
|
||||
fast-path, so rows that go missing from the table are always restored.
|
||||
|
||||
is_tree: doctype is a nested set — insert via db_insert() and rebuild
|
||||
lft/rgt once at the end (avoids per-row tree recomputation and the
|
||||
|
|
@ -102,11 +94,6 @@ def sync_master_data(
|
|||
frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}")
|
||||
return {"skipped": True}
|
||||
|
||||
current_hash = _file_hash(json_path)
|
||||
stored_hash = frappe.db.get_global(_global_key(doctype))
|
||||
if stored_hash == current_hash:
|
||||
return {"unchanged": True}
|
||||
|
||||
target = _load_records(json_path, key_field)
|
||||
if not target:
|
||||
return {"empty_file": True}
|
||||
|
|
@ -195,7 +182,6 @@ def sync_master_data(
|
|||
|
||||
rebuild_tree(doctype)
|
||||
|
||||
frappe.db.set_global(_global_key(doctype), current_hash)
|
||||
frappe.db.commit()
|
||||
|
||||
summary = {
|
||||
|
|
@ -206,10 +192,11 @@ def sync_master_data(
|
|||
"disabled": disabled,
|
||||
"skipped": skipped,
|
||||
}
|
||||
print(
|
||||
f"[taxes_az] {doctype}: "
|
||||
f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
|
||||
)
|
||||
if inserted or updated or deleted or disabled or skipped:
|
||||
print(
|
||||
f"[taxes_az] {doctype}: "
|
||||
f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue