From befe5d6486235fce53cfac40e4648e6912381aff Mon Sep 17 00:00:00 2001 From: Ali <010109ali@gmail.com> Date: Fri, 6 Feb 2026 14:59:12 +0400 Subject: [PATCH] fixed tax article report bugs --- check_tax_articles.py | 141 +++++++++++++++++++++ check_tax_articles_safe.py | 194 +++++++++++++++++++++++++++++ taxes_az/fixtures/tax_article.json | 13 +- taxes_az/normalize_tax_articles.py | 99 ++++++++++++--- 4 files changed, 415 insertions(+), 32 deletions(-) create mode 100644 check_tax_articles.py create mode 100644 check_tax_articles_safe.py diff --git a/check_tax_articles.py b/check_tax_articles.py new file mode 100644 index 0000000..51fe153 --- /dev/null +++ b/check_tax_articles.py @@ -0,0 +1,141 @@ +""" +Tax Articles Verification Script +Run: bench --site [site-name] console +Then copy this code into the console +""" + +import frappe +import unicodedata + +print("=" * 100) +print("TAX ARTICLES VERIFICATION") +print("=" * 100) + +# 1. Check for curly quotes +print("\n1. CURLY QUOTES (should be 0):") +print("-" * 100) + +curly_count = frappe.db.sql(""" + SELECT COUNT(*) as count + FROM `tabTax Article` + WHERE HEX(article_name) LIKE '%E2809C%' + OR HEX(article_name) LIKE '%E2809D%' +""")[0][0] + +if curly_count == 0: + print("✓ No curly quotes found") +else: + print(f"✗ Found {curly_count} articles with curly quotes!") + + # Show problematic articles + articles = frappe.db.sql(""" + SELECT name + FROM `tabTax Article` + WHERE HEX(article_name) LIKE '%E2809C%' + OR HEX(article_name) LIKE '%E2809D%' + LIMIT 5 + """, as_dict=True) + + for a in articles: + print(f" - {a['name'][:70]}...") + +# 2. Check NFD/NFC normalization +print("\n2. NFD/NFC NORMALIZATION:") +print("-" * 100) + +articles = frappe.db.sql(""" + SELECT name, article_name + FROM `tabTax Article` + WHERE article_name IS NOT NULL AND article_name != '' +""", as_dict=True) + +nfd_issues = [] +for article in articles: + original = article['article_name'] + normalized = unicodedata.normalize('NFC', original) + if len(original) != len(normalized): + nfd_issues.append({ + 'name': article['name'], + 'len_original': len(original), + 'len_normalized': len(normalized) + }) + +if not nfd_issues: + print("✓ All articles are in NFC form") +else: + print(f"✗ Found {len(nfd_issues)} articles requiring normalization:") + for issue in nfd_issues[:5]: + print(f" - {issue['name'][:60]}... ({issue['len_original']} → {issue['len_normalized']})") + +# 3. Check matching with report code +print("\n3. MATCHING WITH REPORT CODE:") +print("-" * 100) + +from taxes_az.taxes_az.report.tax_articles_revenue_report.tax_articles_revenue_report import get_tax_articles_mapping + +code_mapping = get_tax_articles_mapping() + +problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51'] +mismatches = [] + +for key in problem_keys: + articles = frappe.db.sql(""" + SELECT name, article_name + FROM `tabTax Article` + WHERE name LIKE %s + """, (f'%{key}%',), as_dict=True) + + for article in articles: + if article['article_name'] not in code_mapping: + mismatches.append({ + 'key': key, + 'name': article['name'], + 'article_name': article['article_name'] + }) + +if not mismatches: + print("✓ All problem articles found in code") +else: + print(f"✗ {len(mismatches)} articles NOT found in code mapping:") + for m in mismatches: + print(f" - {m['key']}: {m['name'][:60]}...") + +# 4. Check data in Sales Invoice Items +print("\n4. DATA IN SALES INVOICE ITEMS:") +print("-" * 100) + +for key in problem_keys: + count = frappe.db.sql(""" + SELECT COUNT(*) as count + FROM `tabSales Invoice Item` sii + JOIN `tabSales Invoice` si ON sii.parent = si.name + WHERE si.docstatus = 1 + AND sii.tax_article LIKE %s + """, (f'%{key}%',))[0][0] + + status = "✓" if count > 0 else "✗" + print(f" {status} {key}: {count} records") + +# 5. Final summary +print("\n" + "=" * 100) +print("SUMMARY:") +print("=" * 100) + +total_issues = curly_count + len(nfd_issues) + len(mismatches) + +if total_issues == 0: + print("✓✓✓ ALL CHECKS PASSED! Tax Articles are OK.") +else: + print(f"⚠ Found {total_issues} issues:") + if curly_count > 0: + print(f" - Curly quotes: {curly_count}") + if nfd_issues: + print(f" - NFD normalization: {len(nfd_issues)}") + if mismatches: + print(f" - Not in code: {len(mismatches)}") + + print("\nRun normalization:") + print("from taxes_az.normalize_tax_articles import normalize_tax_articles") + print("normalize_tax_articles()") + +print("=" * 100) diff --git a/check_tax_articles_safe.py b/check_tax_articles_safe.py new file mode 100644 index 0000000..d99c21f --- /dev/null +++ b/check_tax_articles_safe.py @@ -0,0 +1,194 @@ +""" +Tax Articles Verification Script (Safe version with encoding handling) +Run: bench --site [site-name] console +Then copy this code into the console +""" + +import frappe +import unicodedata + +def safe_str(text): + """Safely convert text to string, handling encoding issues""" + if not text: + return "" + try: + # Try normal encoding + return str(text) + except (UnicodeDecodeError, UnicodeEncodeError): + # If failed, use surrogateescape to handle bad bytes + if isinstance(text, bytes): + return text.decode('utf-8', errors='replace') + return text.encode('utf-8', errors='replace').decode('utf-8') + +print("=" * 100) +print("TAX ARTICLES VERIFICATION (SAFE MODE)") +print("=" * 100) + +# 1. Check for curly quotes +print("\n1. CURLY QUOTES (should be 0):") +print("-" * 100) + +try: + curly_count = frappe.db.sql(""" + SELECT COUNT(*) as count + FROM `tabTax Article` + WHERE HEX(article_name) LIKE '%E2809C%' + OR HEX(article_name) LIKE '%E2809D%' + """)[0][0] + + if curly_count == 0: + print("✓ No curly quotes found") + else: + print(f"✗ Found {curly_count} articles with curly quotes!") + + # Show problematic articles + articles = frappe.db.sql(""" + SELECT name + FROM `tabTax Article` + WHERE HEX(article_name) LIKE '%E2809C%' + OR HEX(article_name) LIKE '%E2809D%' + LIMIT 5 + """, as_dict=True) + + for a in articles: + print(f" - {safe_str(a['name'][:70])}...") +except Exception as e: + print(f"✗ Error checking curly quotes: {e}") + +# 2. Check NFD/NFC normalization +print("\n2. NFD/NFC NORMALIZATION:") +print("-" * 100) + +try: + articles = frappe.db.sql(""" + SELECT name, article_name + FROM `tabTax Article` + WHERE article_name IS NOT NULL AND article_name != '' + """, as_dict=True) + + nfd_issues = [] + encoding_errors = [] + + for article in articles: + try: + original = article['article_name'] + # Try to normalize + normalized = unicodedata.normalize('NFC', original) + if len(original) != len(normalized): + nfd_issues.append({ + 'name': article['name'], + 'len_original': len(original), + 'len_normalized': len(normalized) + }) + except (UnicodeDecodeError, UnicodeEncodeError, TypeError) as e: + encoding_errors.append({ + 'name': article['name'], + 'error': str(e) + }) + + if not nfd_issues and not encoding_errors: + print("✓ All articles are in NFC form") + else: + if nfd_issues: + print(f"✗ Found {len(nfd_issues)} articles requiring normalization:") + for issue in nfd_issues[:5]: + print(f" - {safe_str(issue['name'][:60])}... ({issue['len_original']} → {issue['len_normalized']})") + + if encoding_errors: + print(f"\n⚠ Found {len(encoding_errors)} articles with encoding errors:") + for issue in encoding_errors[:5]: + print(f" - {safe_str(issue['name'][:60])}...") + print(f" Error: {issue['error']}") +except Exception as e: + print(f"✗ Error checking normalization: {e}") + +# 3. Check matching with report code +print("\n3. MATCHING WITH REPORT CODE:") +print("-" * 100) + +try: + from taxes_az.taxes_az.report.tax_articles_revenue_report.tax_articles_revenue_report import get_tax_articles_mapping + + code_mapping = get_tax_articles_mapping() + + problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51'] + mismatches = [] + + for key in problem_keys: + articles = frappe.db.sql(""" + SELECT name, article_name + FROM `tabTax Article` + WHERE name LIKE %s + """, (f'%{key}%',), as_dict=True) + + for article in articles: + try: + if article['article_name'] not in code_mapping: + mismatches.append({ + 'key': key, + 'name': article['name'], + 'article_name': safe_str(article['article_name'][:60]) + }) + except (UnicodeDecodeError, UnicodeEncodeError): + mismatches.append({ + 'key': key, + 'name': article['name'], + 'article_name': '[ENCODING ERROR]' + }) + + if not mismatches: + print("✓ All problem articles found in code") + else: + print(f"✗ {len(mismatches)} articles NOT found in code mapping:") + for m in mismatches: + print(f" - {m['key']}: {safe_str(m['name'][:60])}...") +except Exception as e: + print(f"✗ Error checking code mapping: {e}") + +# 4. Check data in Sales Invoice Items +print("\n4. DATA IN SALES INVOICE ITEMS:") +print("-" * 100) + +try: + problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51'] + + for key in problem_keys: + count = frappe.db.sql(""" + SELECT COUNT(*) as count + FROM `tabSales Invoice Item` sii + JOIN `tabSales Invoice` si ON sii.parent = si.name + WHERE si.docstatus = 1 + AND sii.tax_article LIKE %s + """, (f'%{key}%',))[0][0] + + status = "✓" if count > 0 else "✗" + print(f" {status} {key}: {count} records") +except Exception as e: + print(f"✗ Error checking Sales Invoice Items: {e}") + +# 5. Final summary +print("\n" + "=" * 100) +print("SUMMARY:") +print("=" * 100) + +try: + total_issues = curly_count + len(nfd_issues) + len(mismatches) + + if total_issues == 0: + print("✓✓✓ ALL CHECKS PASSED! Tax Articles are OK.") + else: + print(f"⚠ Found {total_issues} issues:") + if curly_count > 0: + print(f" - Curly quotes: {curly_count}") + if nfd_issues: + print(f" - NFD normalization: {len(nfd_issues)}") + if mismatches: + print(f" - Not in code: {len(mismatches)}") + + print("\nRun normalization:") + print("from taxes_az.normalize_tax_articles import normalize_tax_articles") + print("normalize_tax_articles()") +except Exception as e: + print(f"Error in summary: {e}") + +print("=" * 100) diff --git a/taxes_az/fixtures/tax_article.json b/taxes_az/fixtures/tax_article.json index 656855b..788fd4c 100644 --- a/taxes_az/fixtures/tax_article.json +++ b/taxes_az/fixtures/tax_article.json @@ -10,17 +10,6 @@ "old_parent": null, "parent_tax_article": "Verginin hesablanması" }, - { - "article_name": "VM-nin 164.1.47-ci maddəsinə əsasən bina tikintisi fəaliyyəti ilə məşğul olan şəxslər tərəfindən tikilən binanın yaşayış sahəsinin dövlətə ayrılan hissəsi üzrə dövriyyələr", - "declaration": "Declaration of value added tax", - "docstatus": 0, - "doctype": "Tax Article", - "is_group": 0, - "modified": "2025-08-26 20:33:44.270437", - "name": "VM-nin 164.1.47-ci maddəsinə əsasən bina tikintisi fəaliyyəti ilə məşğul olan şəxslər tərəfindən tikilən binanın yaşayış sahəsinin dövlə...", - "old_parent": null, - "parent_tax_article": "ƏDV-dən azad olunan əməliyyatlar" - }, { "article_name": "Vergi Məcəlləsinin 165.5-ci maddəsinə əsasən ƏDV-si qaytarılan malların geri qaytarılması halları üzrə ƏDV-nin hesablanması (NAĞD ödənişlər)", "declaration": "Declaration of value added tax", @@ -1243,7 +1232,7 @@ "parent_tax_article": "ƏDV-dən azad olunan əməliyyatlar" }, { - "article_name": "VM-nin 164.1.51-ci maddəsinə əsasən \"Tibbi sığorta haqqında\" Azərbaycan Respublikasının Qanununa uyğun olaraq icbari tibbi sığorta fondunun, habelə siyahısı müvafiq icra hakimiyyəti orqanının müəyyən etdiyi orqan (qurum) tərəfindən təsdiq edilən ictimai və sosial məqsədlər üçün yaradılmış fondların vəsaiti hesabına tibbi xidmətlərin göstərilməsi", + "article_name": "VM-nin 164.1.51-1-ci maddəsinə əsasən tədris bazası kimi fəaliyyət göstərən tibb müəssisələrində tibbi xidmətlərin göstərilməsi", "declaration": "Declaration of value added tax", "docstatus": 0, "doctype": "Tax Article", diff --git a/taxes_az/normalize_tax_articles.py b/taxes_az/normalize_tax_articles.py index 110ffe3..77505fc 100644 --- a/taxes_az/normalize_tax_articles.py +++ b/taxes_az/normalize_tax_articles.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Normalize Tax Articles - fix Unicode normalization and curly quotes +Normalize Tax Articles - fix Unicode normalization, curly quotes, and NBSP This script: -1. Replaces typographic (curly) quotes with straight quotes -2. Normalizes Unicode to NFC form (composed characters) +1. Replaces non-breaking spaces (NBSP U+00A0) with regular spaces +2. Replaces typographic (curly) quotes with straight quotes +3. Normalizes Unicode to NFC form (composed characters) +4. Fixes both `name` and `article_name` fields Run after migration to ensure all Tax Article names match the code mapping. """ @@ -16,7 +18,8 @@ import unicodedata def normalize_tax_articles(): """ - Normalize all Tax Article article_name values: + Normalize all Tax Article name and article_name values: + - Replace non-breaking spaces (U+00A0) with regular spaces - Replace typographic quotes (U+201C, U+201D) with straight quotes (") - Normalize Unicode to NFC form """ @@ -26,9 +29,38 @@ def normalize_tax_articles(): return try: - # Step 1: Replace curly quotes with straight quotes using HEX - # E2809C = left curly quote " - # E2809D = right curly quote " + # Step 1: Replace non-breaking spaces (NBSP) with regular spaces + # C2A0 = UTF-8 for U+00A0 (NBSP) + # 20 = regular space + + frappe.db.sql(""" + UPDATE `tabTax Article` + SET article_name = REPLACE(article_name, UNHEX('C2A0'), ' ') + WHERE HEX(article_name) LIKE '%C2A0%' + """) + nbsp_article_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0] + + frappe.db.sql(""" + UPDATE `tabTax Article` + SET name = REPLACE(name, UNHEX('C2A0'), ' ') + WHERE HEX(name) LIKE '%C2A0%' + """) + nbsp_name_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0] + + # Also fix NBSP in Sales Invoice Item tax_article references + if frappe.db.table_exists("Sales Invoice Item"): + frappe.db.sql(""" + UPDATE `tabSales Invoice Item` + SET tax_article = REPLACE(tax_article, UNHEX('C2A0'), ' ') + WHERE HEX(tax_article) LIKE '%C2A0%' + """) + nbsp_ref_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0] + else: + nbsp_ref_fixed = 0 + + # Step 2: Replace curly quotes with straight quotes using HEX + # E2809C = left curly quote \u201c + # E2809D = right curly quote \u201d # 22 = straight quote " frappe.db.sql(""" @@ -43,7 +75,7 @@ def normalize_tax_articles(): quotes_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0] - # Step 2: Normalize Unicode to NFC form + # Step 3: Normalize Unicode to NFC form articles = frappe.db.sql(""" SELECT name, article_name FROM `tabTax Article` @@ -51,22 +83,49 @@ def normalize_tax_articles(): """, as_dict=True) nfc_fixed = 0 - for article in articles: - original = article['article_name'] - normalized = unicodedata.normalize('NFC', original) + encoding_errors = 0 - if original != normalized: - frappe.db.sql(""" - UPDATE `tabTax Article` - SET article_name = %s - WHERE name = %s - """, (normalized, article['name'])) - nfc_fixed += 1 + for article in articles: + try: + original = article['article_name'] + + # Handle potential encoding issues + if isinstance(original, bytes): + original = original.decode('utf-8', errors='replace') + + normalized = unicodedata.normalize('NFC', original) + + if original != normalized: + frappe.db.sql(""" + UPDATE `tabTax Article` + SET article_name = %s + WHERE name = %s + """, (normalized, article['name'])) + nfc_fixed += 1 + + except (UnicodeDecodeError, UnicodeEncodeError, TypeError) as e: + encoding_errors += 1 + frappe.log_error( + f"Encoding error in Tax Article {article['name']}: {str(e)}", + "Tax Article Encoding Error" + ) frappe.db.commit() - if quotes_fixed > 0 or nfc_fixed > 0: - message = f"Tax Articles normalized: {quotes_fixed} curly quotes fixed, {nfc_fixed} NFC normalized" + total_fixed = nbsp_article_fixed + nbsp_name_fixed + nbsp_ref_fixed + quotes_fixed + nfc_fixed + if total_fixed > 0: + parts = [] + if nbsp_article_fixed > 0 or nbsp_name_fixed > 0: + parts.append(f"{nbsp_article_fixed + nbsp_name_fixed} NBSP fixed") + if nbsp_ref_fixed > 0: + parts.append(f"{nbsp_ref_fixed} NBSP in references fixed") + if quotes_fixed > 0: + parts.append(f"{quotes_fixed} curly quotes fixed") + if nfc_fixed > 0: + parts.append(f"{nfc_fixed} NFC normalized") + if encoding_errors > 0: + parts.append(f"{encoding_errors} encoding errors") + message = f"Tax Articles normalized: {', '.join(parts)}" print(message) frappe.log_error(message, "Tax Article Normalization") else: