perf(master-data): remove zombie fixture file and add bulk-insert fast path
The previous commit moved Main type of activity data to master_data/ but left the original fixture file in place — Frappe's sync_fixtures reads every .json in app/fixtures/ regardless of the hooks.py fixtures list, so the file was still being imported on every migrate (3499 DELETE+INSERTs). Delete it. Also adds a bulk-insert fast path (db_insert chunks of 500) to sync_master_data for the large first-run case on fresh sites, so the new sync is faster than the legacy fixture import even when populating an empty table. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
7e372f6f36
commit
a5fbff78eb
File diff suppressed because it is too large
Load Diff
|
|
@ -111,8 +111,31 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
|
|||
|
||||
inserted = updated = deleted = disabled = skipped = 0
|
||||
|
||||
for key, rec in target.items():
|
||||
if key not in existing:
|
||||
to_insert = [(key, rec) for key, rec in target.items() if key not in existing]
|
||||
to_check = [(key, rec) for key, rec in target.items() if key in existing]
|
||||
|
||||
# Bulk-insert fast path for the large first-run case (avoid 10k+ ORM .insert()
|
||||
# round-trips). Uses db_insert() per doc — skips before_insert/after_insert
|
||||
# hooks, which is appropriate for static master data.
|
||||
BULK_THRESHOLD = 100
|
||||
if len(to_insert) >= BULK_THRESHOLD:
|
||||
for key, rec in to_insert:
|
||||
doc = frappe.new_doc(doctype)
|
||||
if key_field == "name":
|
||||
doc.name = key
|
||||
doc.flags.name_set = True
|
||||
else:
|
||||
doc.set(key_field, key)
|
||||
for field in value_fields:
|
||||
if field in rec:
|
||||
doc.set(field, rec[field])
|
||||
doc.db_insert()
|
||||
inserted += 1
|
||||
if inserted % 500 == 0:
|
||||
frappe.db.commit()
|
||||
frappe.db.commit()
|
||||
else:
|
||||
for key, rec in to_insert:
|
||||
doc = frappe.new_doc(doctype)
|
||||
if key_field == "name":
|
||||
doc.name = key
|
||||
|
|
@ -126,8 +149,8 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
|
|||
doc.flags.ignore_links = True
|
||||
doc.insert(ignore_if_duplicate=True)
|
||||
inserted += 1
|
||||
continue
|
||||
|
||||
for key, rec in to_check:
|
||||
db_row = existing[key]
|
||||
diff = {f: rec[f] for f in value_fields if f in rec and (db_row.get(f) or "") != (rec[f] or "")}
|
||||
if diff:
|
||||
|
|
|
|||
Loading…
Reference in New Issue