perf(master-data): remove zombie fixture file and add bulk-insert fast path

The previous commit moved Main type of activity data to master_data/ but left
the original fixture file in place — Frappe's sync_fixtures reads every .json
in app/fixtures/ regardless of the hooks.py fixtures list, so the file was
still being imported on every migrate (3499 DELETE+INSERTs). Delete it.

Also adds a bulk-insert fast path (db_insert chunks of 500) to sync_master_data
for the large first-run case on fresh sites, so the new sync is faster than
the legacy fixture import even when populating an empty table.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ali 2026-05-19 13:48:42 +00:00
parent 7e372f6f36
commit a5fbff78eb
2 changed files with 26 additions and 24498 deletions

File diff suppressed because it is too large Load Diff

View File

@ -111,8 +111,31 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
inserted = updated = deleted = disabled = skipped = 0
for key, rec in target.items():
if key not in existing:
to_insert = [(key, rec) for key, rec in target.items() if key not in existing]
to_check = [(key, rec) for key, rec in target.items() if key in existing]
# Bulk-insert fast path for the large first-run case (avoid 10k+ ORM .insert()
# round-trips). Uses db_insert() per doc — skips before_insert/after_insert
# hooks, which is appropriate for static master data.
BULK_THRESHOLD = 100
if len(to_insert) >= BULK_THRESHOLD:
for key, rec in to_insert:
doc = frappe.new_doc(doctype)
if key_field == "name":
doc.name = key
doc.flags.name_set = True
else:
doc.set(key_field, key)
for field in value_fields:
if field in rec:
doc.set(field, rec[field])
doc.db_insert()
inserted += 1
if inserted % 500 == 0:
frappe.db.commit()
frappe.db.commit()
else:
for key, rec in to_insert:
doc = frappe.new_doc(doctype)
if key_field == "name":
doc.name = key
@ -126,8 +149,8 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
doc.flags.ignore_links = True
doc.insert(ignore_if_duplicate=True)
inserted += 1
continue
for key, rec in to_check:
db_row = existing[key]
diff = {f: rec[f] for f in value_fields if f in rec and (db_row.get(f) or "") != (rec[f] or "")}
if diff: