perf(master-data): sync Item Groups via hash-diff instead of after_migrate loop

create_item_groups() ran a 26-record exists-check loop on every migrate
(the "Группа X уже существует, пропускаем" spam in migrate logs). Replace
with the master_data hash-diff sync: fast-paths via stored file hash, applies
only the delta otherwise.

Item Group is a tree doctype shared with ERPNext/user groups, so the sync
gains two options: is_tree=True (insert via db_insert, rebuild lft/rgt once
at the end) and manage_deletes=False (never delete rows absent from the JSON,
since the JSON is only the AZ tax-specific subset).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

CLAUDE.md

@@ -39,10 +39,12 @@ bench --site [site-name] run-tests --app taxes_az
 
 ### Post-migration Hook
 
-The app automatically creates Item Groups after migration via:
+The app automatically syncs Item Groups after migration via:
 ```python
-after_migrate = "taxes_az.create_item_group.create_item_groups"
+after_migrate = "taxes_az.master_data.sync.sync_item_groups"
 ```
+Item Groups ship as `master_data/item_group.json` and are applied with a
+tree-aware hash-diff sync (insert/update only). See `master_data/sync.py`.
 
 ## Architecture
 
@@ -124,7 +126,7 @@ The app manages specialized item groups for Azərbaycan tax purposes:
 - **Faydalı qazıntılar** (Mining Resources) - 20+ types of minerals with units
 - **İdman, mərc və lotoreya oyunları** (Gaming) - Sports betting and lottery
 
-These are created via `create_item_group.py` after migration.
+These are synced from `master_data/item_group.json` after migration.
 
 ## Frappe Framework Conventions
 
@@ -212,7 +214,7 @@ taxes_az/
 ├── taxes_az/                    # Main module
 │   ├── auth.py                  # E-Taxes authentication
 │   ├── company_etaxes.py        # Company integration
-│   ├── create_item_group.py     # Item group creation
+│   ├── master_data/             # JSON master data + hash-diff sync
 │   ├── item_group.py            # Item group validation
 │   ├── hooks.py                 # Frappe hooks
 │   ├── client/                  # Client scripts

TAX_ARTICLES_NORMALIZATION.md

@@ -60,7 +60,7 @@ NFC: ş (1 символ)
 
 ```python
 after_migrate = [
-    "taxes_az.create_item_group.create_item_groups",
+    "taxes_az.master_data.sync.sync_item_groups",
     "taxes_az.normalize_tax_articles.normalize_on_migrate"
 ]
 ```

taxes_az/create_item_group.py

@@ -1,227 +0,0 @@
-import frappe
-from frappe import _
-
-def create_item_groups():
-    """
-    Создает Item Groups согласно фикстурам
-    """
-    
-    # Определяем возможные названия родительской группы
-    parent_names = ["All Item Groups", "Bütün Element Qrupları"]
-    parent_name = None
-    
-    # Проверяем какое название родительской группы существует
-    for name in parent_names:
-        if frappe.db.exists("Item Group", name):
-            parent_name = name
-            break
-    
-    if not parent_name:
-        print("Родительская группа не найдена. Используем 'All Item Groups'")
-        parent_name = "All Item Groups"
-    
-    # Данные групп из фикстур
-    item_groups = [
-        {
-            "item_group_name": "Neft Məhsulları",
-            "is_group": 1,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Yanacaq növləri", 
-            "is_group": 0,
-            "parent_item_group": "Neft Məhsulları"
-        },
-        {
-            "item_group_name": "Faydalı qazıntılar",
-            "is_group": 1,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Xam neft - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name  # В фикстуре old_parent не указан, используем корневую
-        },
-        {
-            "item_group_name": "Təbii qaz - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Filiz faydalı qazıntıları: bütün növ metallar - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Mişar daşları - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Üzlük daşları - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Seolit - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Barit - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Yüngül doldurucular üçün gillər - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Vulkan külü və pemza - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Kvars qumları - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Sement xammalı - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Duz - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Yodlu bromlu sular - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Mineral sular - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Bentonit gillər - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Kərpic-kirəmit gilləri - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Tikinti qumları - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Çınqıl xammalı - m³",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Gips, gəc - ton",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "İdman, mərc və lotoreya oyunları",
-            "is_group": 1,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "İdman və mərc oyunları",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        },
-        {
-            "item_group_name": "Lotoreya oyunları",
-            "is_group": 0,
-            "parent_item_group": parent_name
-        }
-    ]
-    
-    created_count = 0
-    skipped_count = 0
-    
-    for group_data in item_groups:
-        try:
-            # Проверяем, существует ли уже такая группа
-            if frappe.db.exists("Item Group", group_data["item_group_name"]):
-                print(f"Группа '{group_data['item_group_name']}' уже существует, пропускаем...")
-                skipped_count += 1
-                continue
-            
-            # Создаем новую Item Group
-            item_group = frappe.new_doc("Item Group")
-            item_group.item_group_name = group_data["item_group_name"]
-            item_group.is_group = group_data["is_group"]
-            item_group.parent_item_group = group_data["parent_item_group"]
-            
-            # Сохраняем документ
-            item_group.insert(ignore_permissions=True)
-            frappe.db.commit()
-            
-            print(f"✓ Создана группа: {group_data['item_group_name']}")
-            created_count += 1
-            
-        except Exception as e:
-            print(f"✗ Ошибка при создании группы '{group_data['item_group_name']}': {str(e)}")
-            continue
-    
-    print(f"\nРезультат:")
-    print(f"Создано новых групп: {created_count}")
-    print(f"Пропущено существующих: {skipped_count}")
-    print(f"Всего обработано: {len(item_groups)}")
-
-def delete_all_created_groups():
-    """
-    Функция для удаления всех созданных групп (для отката)
-    ВНИМАНИЕ: Используйте осторожно!
-    """
-    group_names = [
-        "Neft Məhsulları", "Yanacaq növləri", "Faydalı qazıntılar",
-        "Xam neft - ton", "Təbii qaz - m³", 
-        "Filiz faydalı qazıntıları: bütün növ metallar - ton",
-        "Mişar daşları - m³", "Üzlük daşları - m³", "Seolit - ton",
-        "Barit - ton", "Yüngül doldurucular üçün gillər - m³",
-        "Vulkan külü və pemza - m³", "Kvars qumları - ton",
-        "Sement xammalı - ton", "Duz - ton",
-        "Qiymətli və yarımqiymətli bəzək daşları - karat / qram",
-        "Yodlu bromlu sular - m³", "Mineral sular - m³",
-        "Bentonit gillər - ton", "Kərpic-kirəmit gilləri - m³",
-        "Tikinti qumları - m³", "Çınqıl xammalı - m³",
-        "Gips, gəc - ton", "İdman, mərc və lotoreya oyunları",
-        "İdman və mərc oyunları", "Lotoreya oyunları"
-    ]
-    
-    deleted_count = 0
-    
-    for group_name in group_names:
-        try:
-            if frappe.db.exists("Item Group", group_name):
-                frappe.delete_doc("Item Group", group_name)
-                print(f"✓ Удалена группа: {group_name}")
-                deleted_count += 1
-        except Exception as e:
-            print(f"✗ Ошибка при удалении группы '{group_name}': {str(e)}")
-    
-    frappe.db.commit()
-    print(f"\nУдалено групп: {deleted_count}")
-
-# Основная функция для запуска
-if __name__ == "__main__":
-    create_item_groups()

taxes_az/hooks.py

@@ -20,7 +20,7 @@ fixtures = [
 
 after_migrate = [
 	"taxes_az.master_data.sync.sync_main_type_of_activity",
-	"taxes_az.create_item_group.create_item_groups",
+	"taxes_az.master_data.sync.sync_item_groups",
 	"taxes_az.normalize_tax_articles.normalize_on_migrate",
 	"taxes_az.setup_accounts.check_and_create_accounts"
 ]

taxes_az/master_data/item_group.json

@@ -0,0 +1,158 @@
+[
+ {
+  "name": "Neft Məhsulları",
+  "item_group_name": "Neft Məhsulları",
+  "is_group": 1,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Yanacaq növləri",
+  "item_group_name": "Yanacaq növləri",
+  "is_group": 0,
+  "parent_item_group": "Neft Məhsulları"
+ },
+ {
+  "name": "Faydalı qazıntılar",
+  "item_group_name": "Faydalı qazıntılar",
+  "is_group": 1,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Xam neft - ton",
+  "item_group_name": "Xam neft - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Təbii qaz - m³",
+  "item_group_name": "Təbii qaz - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Filiz faydalı qazıntıları: bütün növ metallar - ton",
+  "item_group_name": "Filiz faydalı qazıntıları: bütün növ metallar - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Mişar daşları - m³",
+  "item_group_name": "Mişar daşları - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Üzlük daşları - m³",
+  "item_group_name": "Üzlük daşları - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Seolit - ton",
+  "item_group_name": "Seolit - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Barit - ton",
+  "item_group_name": "Barit - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Yüngül doldurucular üçün gillər - m³",
+  "item_group_name": "Yüngül doldurucular üçün gillər - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Vulkan külü və pemza - m³",
+  "item_group_name": "Vulkan külü və pemza - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Kvars qumları - ton",
+  "item_group_name": "Kvars qumları - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Sement xammalı - ton",
+  "item_group_name": "Sement xammalı - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Duz - ton",
+  "item_group_name": "Duz - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram",
+  "item_group_name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Yodlu bromlu sular - m³",
+  "item_group_name": "Yodlu bromlu sular - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Mineral sular - m³",
+  "item_group_name": "Mineral sular - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Bentonit gillər - ton",
+  "item_group_name": "Bentonit gillər - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Kərpic-kirəmit gilləri - m³",
+  "item_group_name": "Kərpic-kirəmit gilləri - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Tikinti qumları - m³",
+  "item_group_name": "Tikinti qumları - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Çınqıl xammalı - m³",
+  "item_group_name": "Çınqıl xammalı - m³",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Gips, gəc - ton",
+  "item_group_name": "Gips, gəc - ton",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "İdman, mərc və lotoreya oyunları",
+  "item_group_name": "İdman, mərc və lotoreya oyunları",
+  "is_group": 1,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "İdman və mərc oyunları",
+  "item_group_name": "İdman və mərc oyunları",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ },
+ {
+  "name": "Lotoreya oyunları",
+  "item_group_name": "Lotoreya oyunları",
+  "is_group": 0,
+  "parent_item_group": "All Item Groups"
+ }
+]

taxes_az/master_data/sync.py

@@ -78,11 +78,25 @@ def _try_remove(doctype: str, name: str, has_disabled: bool) -> str:
         return "skipped"
 
 
-def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> dict:
+def sync_master_data(
+    doctype: str,
+    json_path: str,
+    key_field: str = "name",
+    *,
+    is_tree: bool = False,
+    manage_deletes: bool = True,
+) -> dict:
     """Sync a doctype from a JSON file via per-record hash-diff.
 
     Fast-path: if the file hash matches what was stored on the previous run,
     return immediately without touching the DB.
+
+    is_tree: doctype is a nested set — insert via db_insert() and rebuild
+        lft/rgt once at the end (avoids per-row tree recomputation and the
+        need to insert parents before children).
+    manage_deletes: when False, rows present in the DB but not in the JSON
+        are left untouched. Use False when the JSON is only a subset of the
+        table (e.g. Item Group also holds ERPNext and user-created groups).
     """
     if not os.path.exists(json_path):
         frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}")
@@ -114,11 +128,13 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
     to_insert = [(key, rec) for key, rec in target.items() if key not in existing]
     to_check = [(key, rec) for key, rec in target.items() if key in existing]
 
-    # Bulk-insert fast path for the large first-run case (avoid 10k+ ORM .insert()
+    # Bulk-insert path: avoids thousands of ORM .insert() round-trips, and for
-    # round-trips). Uses db_insert() per doc — skips before_insert/after_insert
+    # tree doctypes also sidesteps per-row nested-set recomputation (lft/rgt are
-    # hooks, which is appropriate for static master data.
+    # rebuilt once at the end). db_insert() skips before_insert/after_insert
+    # hooks — appropriate for static master data.
     BULK_THRESHOLD = 100
-    if len(to_insert) >= BULK_THRESHOLD:
+    use_bulk = is_tree or len(to_insert) >= BULK_THRESHOLD
+    if use_bulk:
         for key, rec in to_insert:
             doc = frappe.new_doc(doctype)
             if key_field == "name":
@@ -129,6 +145,10 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
             for field in value_fields:
                 if field in rec:
                     doc.set(field, rec[field])
+            if is_tree:
+                # placeholder bounds; rebuild_tree() fixes them after all inserts
+                doc.lft = 0
+                doc.rgt = 0
             doc.db_insert()
             inserted += 1
             if inserted % 500 == 0:
@@ -157,6 +177,9 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
             frappe.db.set_value(doctype, key, diff, update_modified=False)
             updated += 1
 
+    # Delete rows not present in the JSON. Skipped when the JSON is only a
+    # subset of the table.
+    if manage_deletes:
         for key in set(existing.keys()) - set(target.keys()):
             outcome = _try_remove(doctype, key, has_disabled)
             if outcome == "deleted":
@@ -166,6 +189,12 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
             else:
                 skipped += 1
 
+    # Tree doctypes: recompute lft/rgt once for the whole tree.
+    if is_tree and (inserted or updated or deleted):
+        from frappe.utils.nestedset import rebuild_tree
+
+        rebuild_tree(doctype)
+
     frappe.db.set_global(_global_key(doctype), current_hash)
     frappe.db.commit()
 
@@ -187,3 +216,16 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d
 def sync_main_type_of_activity() -> dict:
     path = frappe.get_app_path("taxes_az", "master_data", "main_type_of_activity.json")
     return sync_master_data("Main type of activity", path, key_field="name")
+
+
+def sync_item_groups() -> dict:
+    """Sync the Azerbaijan tax-specific Item Groups.
+
+    Item Group is a tree doctype shared with ERPNext and user-created groups,
+    so the JSON is only a subset: manage_deletes=False keeps everything else
+    untouched, and is_tree=True rebuilds lft/rgt after the diff.
+    """
+    path = frappe.get_app_path("taxes_az", "master_data", "item_group.json")
+    return sync_master_data(
+        "Item Group", path, key_field="name", is_tree=True, manage_deletes=False
+    )