diff --git a/CLAUDE.md b/CLAUDE.md index 97aedf7..4fa8305 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,10 +39,12 @@ bench --site [site-name] run-tests --app taxes_az ### Post-migration Hook -The app automatically creates Item Groups after migration via: +The app automatically syncs Item Groups after migration via: ```python -after_migrate = "taxes_az.create_item_group.create_item_groups" +after_migrate = "taxes_az.master_data.sync.sync_item_groups" ``` +Item Groups ship as `master_data/item_group.json` and are applied with a +tree-aware hash-diff sync (insert/update only). See `master_data/sync.py`. ## Architecture @@ -124,7 +126,7 @@ The app manages specialized item groups for Azərbaycan tax purposes: - **Faydalı qazıntılar** (Mining Resources) - 20+ types of minerals with units - **İdman, mərc və lotoreya oyunları** (Gaming) - Sports betting and lottery -These are created via `create_item_group.py` after migration. +These are synced from `master_data/item_group.json` after migration. ## Frappe Framework Conventions @@ -212,7 +214,7 @@ taxes_az/ ├── taxes_az/ # Main module │ ├── auth.py # E-Taxes authentication │ ├── company_etaxes.py # Company integration -│ ├── create_item_group.py # Item group creation +│ ├── master_data/ # JSON master data + hash-diff sync │ ├── item_group.py # Item group validation │ ├── hooks.py # Frappe hooks │ ├── client/ # Client scripts diff --git a/TAX_ARTICLES_NORMALIZATION.md b/TAX_ARTICLES_NORMALIZATION.md index 430e006..e449909 100644 --- a/TAX_ARTICLES_NORMALIZATION.md +++ b/TAX_ARTICLES_NORMALIZATION.md @@ -60,7 +60,7 @@ NFC: ş (1 символ) ```python after_migrate = [ - "taxes_az.create_item_group.create_item_groups", + "taxes_az.master_data.sync.sync_item_groups", "taxes_az.normalize_tax_articles.normalize_on_migrate" ] ``` diff --git a/taxes_az/create_item_group.py b/taxes_az/create_item_group.py deleted file mode 100644 index 66c8dc0..0000000 --- a/taxes_az/create_item_group.py +++ /dev/null @@ -1,227 +0,0 @@ -import frappe -from frappe import _ - -def create_item_groups(): - """ - Создает Item Groups согласно фикстурам - """ - - # Определяем возможные названия родительской группы - parent_names = ["All Item Groups", "Bütün Element Qrupları"] - parent_name = None - - # Проверяем какое название родительской группы существует - for name in parent_names: - if frappe.db.exists("Item Group", name): - parent_name = name - break - - if not parent_name: - print("Родительская группа не найдена. Используем 'All Item Groups'") - parent_name = "All Item Groups" - - # Данные групп из фикстур - item_groups = [ - { - "item_group_name": "Neft Məhsulları", - "is_group": 1, - "parent_item_group": parent_name - }, - { - "item_group_name": "Yanacaq növləri", - "is_group": 0, - "parent_item_group": "Neft Məhsulları" - }, - { - "item_group_name": "Faydalı qazıntılar", - "is_group": 1, - "parent_item_group": parent_name - }, - { - "item_group_name": "Xam neft - ton", - "is_group": 0, - "parent_item_group": parent_name # В фикстуре old_parent не указан, используем корневую - }, - { - "item_group_name": "Təbii qaz - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Filiz faydalı qazıntıları: bütün növ metallar - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Mişar daşları - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Üzlük daşları - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Seolit - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Barit - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Yüngül doldurucular üçün gillər - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Vulkan külü və pemza - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Kvars qumları - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Sement xammalı - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Duz - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Yodlu bromlu sular - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Mineral sular - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Bentonit gillər - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Kərpic-kirəmit gilləri - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Tikinti qumları - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Çınqıl xammalı - m³", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Gips, gəc - ton", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "İdman, mərc və lotoreya oyunları", - "is_group": 1, - "parent_item_group": parent_name - }, - { - "item_group_name": "İdman və mərc oyunları", - "is_group": 0, - "parent_item_group": parent_name - }, - { - "item_group_name": "Lotoreya oyunları", - "is_group": 0, - "parent_item_group": parent_name - } - ] - - created_count = 0 - skipped_count = 0 - - for group_data in item_groups: - try: - # Проверяем, существует ли уже такая группа - if frappe.db.exists("Item Group", group_data["item_group_name"]): - print(f"Группа '{group_data['item_group_name']}' уже существует, пропускаем...") - skipped_count += 1 - continue - - # Создаем новую Item Group - item_group = frappe.new_doc("Item Group") - item_group.item_group_name = group_data["item_group_name"] - item_group.is_group = group_data["is_group"] - item_group.parent_item_group = group_data["parent_item_group"] - - # Сохраняем документ - item_group.insert(ignore_permissions=True) - frappe.db.commit() - - print(f"✓ Создана группа: {group_data['item_group_name']}") - created_count += 1 - - except Exception as e: - print(f"✗ Ошибка при создании группы '{group_data['item_group_name']}': {str(e)}") - continue - - print(f"\nРезультат:") - print(f"Создано новых групп: {created_count}") - print(f"Пропущено существующих: {skipped_count}") - print(f"Всего обработано: {len(item_groups)}") - -def delete_all_created_groups(): - """ - Функция для удаления всех созданных групп (для отката) - ВНИМАНИЕ: Используйте осторожно! - """ - group_names = [ - "Neft Məhsulları", "Yanacaq növləri", "Faydalı qazıntılar", - "Xam neft - ton", "Təbii qaz - m³", - "Filiz faydalı qazıntıları: bütün növ metallar - ton", - "Mişar daşları - m³", "Üzlük daşları - m³", "Seolit - ton", - "Barit - ton", "Yüngül doldurucular üçün gillər - m³", - "Vulkan külü və pemza - m³", "Kvars qumları - ton", - "Sement xammalı - ton", "Duz - ton", - "Qiymətli və yarımqiymətli bəzək daşları - karat / qram", - "Yodlu bromlu sular - m³", "Mineral sular - m³", - "Bentonit gillər - ton", "Kərpic-kirəmit gilləri - m³", - "Tikinti qumları - m³", "Çınqıl xammalı - m³", - "Gips, gəc - ton", "İdman, mərc və lotoreya oyunları", - "İdman və mərc oyunları", "Lotoreya oyunları" - ] - - deleted_count = 0 - - for group_name in group_names: - try: - if frappe.db.exists("Item Group", group_name): - frappe.delete_doc("Item Group", group_name) - print(f"✓ Удалена группа: {group_name}") - deleted_count += 1 - except Exception as e: - print(f"✗ Ошибка при удалении группы '{group_name}': {str(e)}") - - frappe.db.commit() - print(f"\nУдалено групп: {deleted_count}") - -# Основная функция для запуска -if __name__ == "__main__": - create_item_groups() \ No newline at end of file diff --git a/taxes_az/hooks.py b/taxes_az/hooks.py index 6da9d89..eb2eccc 100644 --- a/taxes_az/hooks.py +++ b/taxes_az/hooks.py @@ -20,7 +20,7 @@ fixtures = [ after_migrate = [ "taxes_az.master_data.sync.sync_main_type_of_activity", - "taxes_az.create_item_group.create_item_groups", + "taxes_az.master_data.sync.sync_item_groups", "taxes_az.normalize_tax_articles.normalize_on_migrate", "taxes_az.setup_accounts.check_and_create_accounts" ] diff --git a/taxes_az/master_data/item_group.json b/taxes_az/master_data/item_group.json new file mode 100644 index 0000000..6a4c93b --- /dev/null +++ b/taxes_az/master_data/item_group.json @@ -0,0 +1,158 @@ +[ + { + "name": "Neft Məhsulları", + "item_group_name": "Neft Məhsulları", + "is_group": 1, + "parent_item_group": "All Item Groups" + }, + { + "name": "Yanacaq növləri", + "item_group_name": "Yanacaq növləri", + "is_group": 0, + "parent_item_group": "Neft Məhsulları" + }, + { + "name": "Faydalı qazıntılar", + "item_group_name": "Faydalı qazıntılar", + "is_group": 1, + "parent_item_group": "All Item Groups" + }, + { + "name": "Xam neft - ton", + "item_group_name": "Xam neft - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Təbii qaz - m³", + "item_group_name": "Təbii qaz - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Filiz faydalı qazıntıları: bütün növ metallar - ton", + "item_group_name": "Filiz faydalı qazıntıları: bütün növ metallar - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Mişar daşları - m³", + "item_group_name": "Mişar daşları - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Üzlük daşları - m³", + "item_group_name": "Üzlük daşları - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Seolit - ton", + "item_group_name": "Seolit - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Barit - ton", + "item_group_name": "Barit - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Yüngül doldurucular üçün gillər - m³", + "item_group_name": "Yüngül doldurucular üçün gillər - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Vulkan külü və pemza - m³", + "item_group_name": "Vulkan külü və pemza - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Kvars qumları - ton", + "item_group_name": "Kvars qumları - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Sement xammalı - ton", + "item_group_name": "Sement xammalı - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Duz - ton", + "item_group_name": "Duz - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram", + "item_group_name": "Qiymətli və yarımqiymətli bəzək daşları - karat / qram", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Yodlu bromlu sular - m³", + "item_group_name": "Yodlu bromlu sular - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Mineral sular - m³", + "item_group_name": "Mineral sular - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Bentonit gillər - ton", + "item_group_name": "Bentonit gillər - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Kərpic-kirəmit gilləri - m³", + "item_group_name": "Kərpic-kirəmit gilləri - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Tikinti qumları - m³", + "item_group_name": "Tikinti qumları - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Çınqıl xammalı - m³", + "item_group_name": "Çınqıl xammalı - m³", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Gips, gəc - ton", + "item_group_name": "Gips, gəc - ton", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "İdman, mərc və lotoreya oyunları", + "item_group_name": "İdman, mərc və lotoreya oyunları", + "is_group": 1, + "parent_item_group": "All Item Groups" + }, + { + "name": "İdman və mərc oyunları", + "item_group_name": "İdman və mərc oyunları", + "is_group": 0, + "parent_item_group": "All Item Groups" + }, + { + "name": "Lotoreya oyunları", + "item_group_name": "Lotoreya oyunları", + "is_group": 0, + "parent_item_group": "All Item Groups" + } +] \ No newline at end of file diff --git a/taxes_az/master_data/sync.py b/taxes_az/master_data/sync.py index b94b67d..00b1499 100644 --- a/taxes_az/master_data/sync.py +++ b/taxes_az/master_data/sync.py @@ -78,11 +78,25 @@ def _try_remove(doctype: str, name: str, has_disabled: bool) -> str: return "skipped" -def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> dict: +def sync_master_data( + doctype: str, + json_path: str, + key_field: str = "name", + *, + is_tree: bool = False, + manage_deletes: bool = True, +) -> dict: """Sync a doctype from a JSON file via per-record hash-diff. Fast-path: if the file hash matches what was stored on the previous run, return immediately without touching the DB. + + is_tree: doctype is a nested set — insert via db_insert() and rebuild + lft/rgt once at the end (avoids per-row tree recomputation and the + need to insert parents before children). + manage_deletes: when False, rows present in the DB but not in the JSON + are left untouched. Use False when the JSON is only a subset of the + table (e.g. Item Group also holds ERPNext and user-created groups). """ if not os.path.exists(json_path): frappe.logger().warning(f"[taxes_az] master data file not found: {json_path}") @@ -114,11 +128,13 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d to_insert = [(key, rec) for key, rec in target.items() if key not in existing] to_check = [(key, rec) for key, rec in target.items() if key in existing] - # Bulk-insert fast path for the large first-run case (avoid 10k+ ORM .insert() - # round-trips). Uses db_insert() per doc — skips before_insert/after_insert - # hooks, which is appropriate for static master data. + # Bulk-insert path: avoids thousands of ORM .insert() round-trips, and for + # tree doctypes also sidesteps per-row nested-set recomputation (lft/rgt are + # rebuilt once at the end). db_insert() skips before_insert/after_insert + # hooks — appropriate for static master data. BULK_THRESHOLD = 100 - if len(to_insert) >= BULK_THRESHOLD: + use_bulk = is_tree or len(to_insert) >= BULK_THRESHOLD + if use_bulk: for key, rec in to_insert: doc = frappe.new_doc(doctype) if key_field == "name": @@ -129,6 +145,10 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d for field in value_fields: if field in rec: doc.set(field, rec[field]) + if is_tree: + # placeholder bounds; rebuild_tree() fixes them after all inserts + doc.lft = 0 + doc.rgt = 0 doc.db_insert() inserted += 1 if inserted % 500 == 0: @@ -157,14 +177,23 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d frappe.db.set_value(doctype, key, diff, update_modified=False) updated += 1 - for key in set(existing.keys()) - set(target.keys()): - outcome = _try_remove(doctype, key, has_disabled) - if outcome == "deleted": - deleted += 1 - elif outcome == "disabled": - disabled += 1 - else: - skipped += 1 + # Delete rows not present in the JSON. Skipped when the JSON is only a + # subset of the table. + if manage_deletes: + for key in set(existing.keys()) - set(target.keys()): + outcome = _try_remove(doctype, key, has_disabled) + if outcome == "deleted": + deleted += 1 + elif outcome == "disabled": + disabled += 1 + else: + skipped += 1 + + # Tree doctypes: recompute lft/rgt once for the whole tree. + if is_tree and (inserted or updated or deleted): + from frappe.utils.nestedset import rebuild_tree + + rebuild_tree(doctype) frappe.db.set_global(_global_key(doctype), current_hash) frappe.db.commit() @@ -187,3 +216,16 @@ def sync_master_data(doctype: str, json_path: str, key_field: str = "name") -> d def sync_main_type_of_activity() -> dict: path = frappe.get_app_path("taxes_az", "master_data", "main_type_of_activity.json") return sync_master_data("Main type of activity", path, key_field="name") + + +def sync_item_groups() -> dict: + """Sync the Azerbaijan tax-specific Item Groups. + + Item Group is a tree doctype shared with ERPNext and user-created groups, + so the JSON is only a subset: manage_deletes=False keeps everything else + untouched, and is_tree=True rebuilds lft/rgt after the diff. + """ + path = frappe.get_app_path("taxes_az", "master_data", "item_group.json") + return sync_master_data( + "Item Group", path, key_field="name", is_tree=True, manage_deletes=False + )