perf(master-data): sync Main type of activity via hash-diff instead of fixture

Fixture import does DELETE+INSERT for every row on every migrate (force=True
in frappe's import_doc), so 3499 rows of Main type of activity added minutes
to each migrate. Replace with a hash-diff sync that fast-paths via stored
file hash and otherwise applies only the delta (insert new, update changed,
smart-delete removed with disabled-fallback).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

taxes_az/hooks.py

@@ -12,17 +12,14 @@ fixtures = [
             ["module", "=", "Taxes Az"]
         ]
     },
-    {
-        "doctype": "Main type of activity",
-        "filters": []  
-    },
     {
         "doctype": "Business Classification",
-        "filters": []  
+        "filters": []
     }
 ]
 
 after_migrate = [
+	"taxes_az.master_data.sync.sync_main_type_of_activity",
 	"taxes_az.create_item_group.create_item_groups",
 	"taxes_az.normalize_tax_articles.normalize_on_migrate",
 	"taxes_az.setup_accounts.check_and_create_accounts"

taxes_az/master_data/sync.py

@@ -0,0 +1,166 @@
+"""Hash-diff sync for master data shipped as JSON.
+
+The legacy approach (fixtures hook) re-imports every record on every `bench
+migrate` via DELETE+INSERT. For static reference data with thousands of rows
+that's a multi-minute hit. This module compares the file hash first (fast-path
+exit when unchanged) and otherwise applies only the diff: insert new rows,
+update changed rows, delete rows missing from the file (or disable / skip when
+they are referenced by existing documents).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterable
+
+import frappe
+
+
+def _file_hash(path: str) -> str:
+    h = hashlib.md5(usedforsecurity=False)
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8192), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _global_key(doctype: str) -> str:
+    return f"taxes_az:master_data_hash:{frappe.scrub(doctype)}"
+
+
+def _load_records(path: str, key_field: str) -> dict[str, dict]:
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+    if not isinstance(raw, list):
+        frappe.throw(f"{path}: expected a JSON array")
+
+    out: dict[str, dict] = {}
+    for rec in raw:
+        key = rec.get(key_field)
+        if not key:
+            continue
+        out[str(key)] = rec
+    return out
+
+
+def _fetch_existing(doctype: str, fields: Iterable[str], key_field: str) -> dict[str, dict]:
+    rows = frappe.db.sql(
+        f"SELECT {', '.join('`' + f + '`' for f in fields)} FROM `tab{doctype}`",
+        as_dict=True,
+    )
+    return {str(row[key_field]): row for row in rows}
+
+
+def _try_remove(doctype: str, name: str, has_disabled: bool) -> str:
+    """Smart-delete: try hard delete; on link conflicts soft-disable or skip.
+
+    Returns one of: "deleted", "disabled", "skipped".
+    """
+    try:
+        frappe.delete_doc(
+            doctype,
+            name,
+            ignore_permissions=True,
+            ignore_missing=True,
+            delete_permanently=True,
+        )
+        return "deleted"
+    except frappe.LinkExistsError:
+        if has_disabled:
+            frappe.db.set_value(doctype, name, "disabled", 1, update_modified=False)
+            return "disabled"
+        frappe.logger().warning(
+            f"[taxes_az] {doctype} '{name}' is referenced; "
+            f"cannot delete and no 'disabled' field — kept as-is"
+        )
+        return "skipped"
+
+
+def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> dict:
+    """Sync a doctype from a JSON file via per-record hash-diff.
+
+    Fast-path: if the file hash matches what was stored on the previous run,
+    return immediately without touching the DB.
+    """
+    if not os.path.exists(json_path):
+        frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}")
+        return {"skipped": True}
+
+    current_hash = _file_hash(json_path)
+    stored_hash = frappe.db.get_global(_global_key(doctype))
+    if stored_hash == current_hash:
+        return {"unchanged": True}
+
+    target = _load_records(json_path, key_field)
+    if not target:
+        return {"empty_file": True}
+
+    meta = frappe.get_meta(doctype)
+    # value-bearing fields actually present in the JSON, minus the key itself
+    sample = next(iter(target.values()))
+    value_fields = [
+        f for f in sample.keys()
+        if f != key_field and (meta.has_field(f) or f == "name")
+    ]
+    select_fields = [key_field, *value_fields] if key_field not in value_fields else value_fields
+    existing = _fetch_existing(doctype, select_fields, key_field)
+
+    has_disabled = meta.has_field("disabled")
+
+    inserted = updated = deleted = disabled = skipped = 0
+
+    for key, rec in target.items():
+        if key not in existing:
+            doc = frappe.new_doc(doctype)
+            if key_field == "name":
+                doc.name = key
+            else:
+                doc.set(key_field, key)
+            for field in value_fields:
+                if field in rec:
+                    doc.set(field, rec[field])
+            doc.flags.ignore_permissions = True
+            doc.flags.ignore_mandatory = True
+            doc.flags.ignore_links = True
+            doc.insert(ignore_if_duplicate=True)
+            inserted += 1
+            continue
+
+        db_row = existing[key]
+        diff = {f: rec[f] for f in value_fields if f in rec and (db_row.get(f) or "") != (rec[f] or "")}
+        if diff:
+            frappe.db.set_value(doctype, key, diff, update_modified=False)
+            updated += 1
+
+    for key in set(existing.keys()) - set(target.keys()):
+        outcome = _try_remove(doctype, key, has_disabled)
+        if outcome == "deleted":
+            deleted += 1
+        elif outcome == "disabled":
+            disabled += 1
+        else:
+            skipped += 1
+
+    frappe.db.set_global(_global_key(doctype), current_hash)
+    frappe.db.commit()
+
+    summary = {
+        "doctype": doctype,
+        "inserted": inserted,
+        "updated": updated,
+        "deleted": deleted,
+        "disabled": disabled,
+        "skipped": skipped,
+    }
+    print(
+        f"[taxes_az] {doctype}: "
+        f"+{inserted} ~{updated} -{deleted} (disabled {disabled}, skipped {skipped})"
+    )
+    return summary
+
+
+def sync_main_type_of_activity() -> dict:
+    path = frappe.get_app_path("taxes_az", "master_data", "main_type_of_activity.json")
+    return sync_master_data("Main type of activity", path, key_field="name")