fixed tax article report bugs
This commit is contained in:
parent
2795ce869b
commit
befe5d6486
|
|
@ -0,0 +1,141 @@
|
||||||
|
"""
|
||||||
|
Tax Articles Verification Script
|
||||||
|
Run: bench --site [site-name] console
|
||||||
|
Then copy this code into the console
|
||||||
|
"""
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
print("=" * 100)
|
||||||
|
print("TAX ARTICLES VERIFICATION")
|
||||||
|
print("=" * 100)
|
||||||
|
|
||||||
|
# 1. Check for curly quotes
|
||||||
|
print("\n1. CURLY QUOTES (should be 0):")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
curly_count = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) as count
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE HEX(article_name) LIKE '%E2809C%'
|
||||||
|
OR HEX(article_name) LIKE '%E2809D%'
|
||||||
|
""")[0][0]
|
||||||
|
|
||||||
|
if curly_count == 0:
|
||||||
|
print("✓ No curly quotes found")
|
||||||
|
else:
|
||||||
|
print(f"✗ Found {curly_count} articles with curly quotes!")
|
||||||
|
|
||||||
|
# Show problematic articles
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE HEX(article_name) LIKE '%E2809C%'
|
||||||
|
OR HEX(article_name) LIKE '%E2809D%'
|
||||||
|
LIMIT 5
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
for a in articles:
|
||||||
|
print(f" - {a['name'][:70]}...")
|
||||||
|
|
||||||
|
# 2. Check NFD/NFC normalization
|
||||||
|
print("\n2. NFD/NFC NORMALIZATION:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name, article_name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE article_name IS NOT NULL AND article_name != ''
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
nfd_issues = []
|
||||||
|
for article in articles:
|
||||||
|
original = article['article_name']
|
||||||
|
normalized = unicodedata.normalize('NFC', original)
|
||||||
|
if len(original) != len(normalized):
|
||||||
|
nfd_issues.append({
|
||||||
|
'name': article['name'],
|
||||||
|
'len_original': len(original),
|
||||||
|
'len_normalized': len(normalized)
|
||||||
|
})
|
||||||
|
|
||||||
|
if not nfd_issues:
|
||||||
|
print("✓ All articles are in NFC form")
|
||||||
|
else:
|
||||||
|
print(f"✗ Found {len(nfd_issues)} articles requiring normalization:")
|
||||||
|
for issue in nfd_issues[:5]:
|
||||||
|
print(f" - {issue['name'][:60]}... ({issue['len_original']} → {issue['len_normalized']})")
|
||||||
|
|
||||||
|
# 3. Check matching with report code
|
||||||
|
print("\n3. MATCHING WITH REPORT CODE:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
from taxes_az.taxes_az.report.tax_articles_revenue_report.tax_articles_revenue_report import get_tax_articles_mapping
|
||||||
|
|
||||||
|
code_mapping = get_tax_articles_mapping()
|
||||||
|
|
||||||
|
problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51']
|
||||||
|
mismatches = []
|
||||||
|
|
||||||
|
for key in problem_keys:
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name, article_name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE name LIKE %s
|
||||||
|
""", (f'%{key}%',), as_dict=True)
|
||||||
|
|
||||||
|
for article in articles:
|
||||||
|
if article['article_name'] not in code_mapping:
|
||||||
|
mismatches.append({
|
||||||
|
'key': key,
|
||||||
|
'name': article['name'],
|
||||||
|
'article_name': article['article_name']
|
||||||
|
})
|
||||||
|
|
||||||
|
if not mismatches:
|
||||||
|
print("✓ All problem articles found in code")
|
||||||
|
else:
|
||||||
|
print(f"✗ {len(mismatches)} articles NOT found in code mapping:")
|
||||||
|
for m in mismatches:
|
||||||
|
print(f" - {m['key']}: {m['name'][:60]}...")
|
||||||
|
|
||||||
|
# 4. Check data in Sales Invoice Items
|
||||||
|
print("\n4. DATA IN SALES INVOICE ITEMS:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
for key in problem_keys:
|
||||||
|
count = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) as count
|
||||||
|
FROM `tabSales Invoice Item` sii
|
||||||
|
JOIN `tabSales Invoice` si ON sii.parent = si.name
|
||||||
|
WHERE si.docstatus = 1
|
||||||
|
AND sii.tax_article LIKE %s
|
||||||
|
""", (f'%{key}%',))[0][0]
|
||||||
|
|
||||||
|
status = "✓" if count > 0 else "✗"
|
||||||
|
print(f" {status} {key}: {count} records")
|
||||||
|
|
||||||
|
# 5. Final summary
|
||||||
|
print("\n" + "=" * 100)
|
||||||
|
print("SUMMARY:")
|
||||||
|
print("=" * 100)
|
||||||
|
|
||||||
|
total_issues = curly_count + len(nfd_issues) + len(mismatches)
|
||||||
|
|
||||||
|
if total_issues == 0:
|
||||||
|
print("✓✓✓ ALL CHECKS PASSED! Tax Articles are OK.")
|
||||||
|
else:
|
||||||
|
print(f"⚠ Found {total_issues} issues:")
|
||||||
|
if curly_count > 0:
|
||||||
|
print(f" - Curly quotes: {curly_count}")
|
||||||
|
if nfd_issues:
|
||||||
|
print(f" - NFD normalization: {len(nfd_issues)}")
|
||||||
|
if mismatches:
|
||||||
|
print(f" - Not in code: {len(mismatches)}")
|
||||||
|
|
||||||
|
print("\nRun normalization:")
|
||||||
|
print("from taxes_az.normalize_tax_articles import normalize_tax_articles")
|
||||||
|
print("normalize_tax_articles()")
|
||||||
|
|
||||||
|
print("=" * 100)
|
||||||
|
|
@ -0,0 +1,194 @@
|
||||||
|
"""
|
||||||
|
Tax Articles Verification Script (Safe version with encoding handling)
|
||||||
|
Run: bench --site [site-name] console
|
||||||
|
Then copy this code into the console
|
||||||
|
"""
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
def safe_str(text):
|
||||||
|
"""Safely convert text to string, handling encoding issues"""
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
# Try normal encoding
|
||||||
|
return str(text)
|
||||||
|
except (UnicodeDecodeError, UnicodeEncodeError):
|
||||||
|
# If failed, use surrogateescape to handle bad bytes
|
||||||
|
if isinstance(text, bytes):
|
||||||
|
return text.decode('utf-8', errors='replace')
|
||||||
|
return text.encode('utf-8', errors='replace').decode('utf-8')
|
||||||
|
|
||||||
|
print("=" * 100)
|
||||||
|
print("TAX ARTICLES VERIFICATION (SAFE MODE)")
|
||||||
|
print("=" * 100)
|
||||||
|
|
||||||
|
# 1. Check for curly quotes
|
||||||
|
print("\n1. CURLY QUOTES (should be 0):")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
try:
|
||||||
|
curly_count = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) as count
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE HEX(article_name) LIKE '%E2809C%'
|
||||||
|
OR HEX(article_name) LIKE '%E2809D%'
|
||||||
|
""")[0][0]
|
||||||
|
|
||||||
|
if curly_count == 0:
|
||||||
|
print("✓ No curly quotes found")
|
||||||
|
else:
|
||||||
|
print(f"✗ Found {curly_count} articles with curly quotes!")
|
||||||
|
|
||||||
|
# Show problematic articles
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE HEX(article_name) LIKE '%E2809C%'
|
||||||
|
OR HEX(article_name) LIKE '%E2809D%'
|
||||||
|
LIMIT 5
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
for a in articles:
|
||||||
|
print(f" - {safe_str(a['name'][:70])}...")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"✗ Error checking curly quotes: {e}")
|
||||||
|
|
||||||
|
# 2. Check NFD/NFC normalization
|
||||||
|
print("\n2. NFD/NFC NORMALIZATION:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
try:
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name, article_name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE article_name IS NOT NULL AND article_name != ''
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
nfd_issues = []
|
||||||
|
encoding_errors = []
|
||||||
|
|
||||||
|
for article in articles:
|
||||||
|
try:
|
||||||
|
original = article['article_name']
|
||||||
|
# Try to normalize
|
||||||
|
normalized = unicodedata.normalize('NFC', original)
|
||||||
|
if len(original) != len(normalized):
|
||||||
|
nfd_issues.append({
|
||||||
|
'name': article['name'],
|
||||||
|
'len_original': len(original),
|
||||||
|
'len_normalized': len(normalized)
|
||||||
|
})
|
||||||
|
except (UnicodeDecodeError, UnicodeEncodeError, TypeError) as e:
|
||||||
|
encoding_errors.append({
|
||||||
|
'name': article['name'],
|
||||||
|
'error': str(e)
|
||||||
|
})
|
||||||
|
|
||||||
|
if not nfd_issues and not encoding_errors:
|
||||||
|
print("✓ All articles are in NFC form")
|
||||||
|
else:
|
||||||
|
if nfd_issues:
|
||||||
|
print(f"✗ Found {len(nfd_issues)} articles requiring normalization:")
|
||||||
|
for issue in nfd_issues[:5]:
|
||||||
|
print(f" - {safe_str(issue['name'][:60])}... ({issue['len_original']} → {issue['len_normalized']})")
|
||||||
|
|
||||||
|
if encoding_errors:
|
||||||
|
print(f"\n⚠ Found {len(encoding_errors)} articles with encoding errors:")
|
||||||
|
for issue in encoding_errors[:5]:
|
||||||
|
print(f" - {safe_str(issue['name'][:60])}...")
|
||||||
|
print(f" Error: {issue['error']}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"✗ Error checking normalization: {e}")
|
||||||
|
|
||||||
|
# 3. Check matching with report code
|
||||||
|
print("\n3. MATCHING WITH REPORT CODE:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from taxes_az.taxes_az.report.tax_articles_revenue_report.tax_articles_revenue_report import get_tax_articles_mapping
|
||||||
|
|
||||||
|
code_mapping = get_tax_articles_mapping()
|
||||||
|
|
||||||
|
problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51']
|
||||||
|
mismatches = []
|
||||||
|
|
||||||
|
for key in problem_keys:
|
||||||
|
articles = frappe.db.sql("""
|
||||||
|
SELECT name, article_name
|
||||||
|
FROM `tabTax Article`
|
||||||
|
WHERE name LIKE %s
|
||||||
|
""", (f'%{key}%',), as_dict=True)
|
||||||
|
|
||||||
|
for article in articles:
|
||||||
|
try:
|
||||||
|
if article['article_name'] not in code_mapping:
|
||||||
|
mismatches.append({
|
||||||
|
'key': key,
|
||||||
|
'name': article['name'],
|
||||||
|
'article_name': safe_str(article['article_name'][:60])
|
||||||
|
})
|
||||||
|
except (UnicodeDecodeError, UnicodeEncodeError):
|
||||||
|
mismatches.append({
|
||||||
|
'key': key,
|
||||||
|
'name': article['name'],
|
||||||
|
'article_name': '[ENCODING ERROR]'
|
||||||
|
})
|
||||||
|
|
||||||
|
if not mismatches:
|
||||||
|
print("✓ All problem articles found in code")
|
||||||
|
else:
|
||||||
|
print(f"✗ {len(mismatches)} articles NOT found in code mapping:")
|
||||||
|
for m in mismatches:
|
||||||
|
print(f" - {m['key']}: {safe_str(m['name'][:60])}...")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"✗ Error checking code mapping: {e}")
|
||||||
|
|
||||||
|
# 4. Check data in Sales Invoice Items
|
||||||
|
print("\n4. DATA IN SALES INVOICE ITEMS:")
|
||||||
|
print("-" * 100)
|
||||||
|
|
||||||
|
try:
|
||||||
|
problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51']
|
||||||
|
|
||||||
|
for key in problem_keys:
|
||||||
|
count = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) as count
|
||||||
|
FROM `tabSales Invoice Item` sii
|
||||||
|
JOIN `tabSales Invoice` si ON sii.parent = si.name
|
||||||
|
WHERE si.docstatus = 1
|
||||||
|
AND sii.tax_article LIKE %s
|
||||||
|
""", (f'%{key}%',))[0][0]
|
||||||
|
|
||||||
|
status = "✓" if count > 0 else "✗"
|
||||||
|
print(f" {status} {key}: {count} records")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"✗ Error checking Sales Invoice Items: {e}")
|
||||||
|
|
||||||
|
# 5. Final summary
|
||||||
|
print("\n" + "=" * 100)
|
||||||
|
print("SUMMARY:")
|
||||||
|
print("=" * 100)
|
||||||
|
|
||||||
|
try:
|
||||||
|
total_issues = curly_count + len(nfd_issues) + len(mismatches)
|
||||||
|
|
||||||
|
if total_issues == 0:
|
||||||
|
print("✓✓✓ ALL CHECKS PASSED! Tax Articles are OK.")
|
||||||
|
else:
|
||||||
|
print(f"⚠ Found {total_issues} issues:")
|
||||||
|
if curly_count > 0:
|
||||||
|
print(f" - Curly quotes: {curly_count}")
|
||||||
|
if nfd_issues:
|
||||||
|
print(f" - NFD normalization: {len(nfd_issues)}")
|
||||||
|
if mismatches:
|
||||||
|
print(f" - Not in code: {len(mismatches)}")
|
||||||
|
|
||||||
|
print("\nRun normalization:")
|
||||||
|
print("from taxes_az.normalize_tax_articles import normalize_tax_articles")
|
||||||
|
print("normalize_tax_articles()")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error in summary: {e}")
|
||||||
|
|
||||||
|
print("=" * 100)
|
||||||
|
|
@ -10,17 +10,6 @@
|
||||||
"old_parent": null,
|
"old_parent": null,
|
||||||
"parent_tax_article": "Verginin hesablanması"
|
"parent_tax_article": "Verginin hesablanması"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"article_name": "VM-nin 164.1.47-ci maddəsinə əsasən bina tikintisi fəaliyyəti ilə məşğul olan şəxslər tərəfindən tikilən binanın yaşayış sahəsinin dövlətə ayrılan hissəsi üzrə dövriyyələr",
|
|
||||||
"declaration": "Declaration of value added tax",
|
|
||||||
"docstatus": 0,
|
|
||||||
"doctype": "Tax Article",
|
|
||||||
"is_group": 0,
|
|
||||||
"modified": "2025-08-26 20:33:44.270437",
|
|
||||||
"name": "VM-nin 164.1.47-ci maddəsinə əsasən bina tikintisi fəaliyyəti ilə məşğul olan şəxslər tərəfindən tikilən binanın yaşayış sahəsinin dövlə...",
|
|
||||||
"old_parent": null,
|
|
||||||
"parent_tax_article": "ƏDV-dən azad olunan əməliyyatlar"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"article_name": "Vergi Məcəlləsinin 165.5-ci maddəsinə əsasən ƏDV-si qaytarılan malların geri qaytarılması halları üzrə ƏDV-nin hesablanması (NAĞD ödənişlər)",
|
"article_name": "Vergi Məcəlləsinin 165.5-ci maddəsinə əsasən ƏDV-si qaytarılan malların geri qaytarılması halları üzrə ƏDV-nin hesablanması (NAĞD ödənişlər)",
|
||||||
"declaration": "Declaration of value added tax",
|
"declaration": "Declaration of value added tax",
|
||||||
|
|
@ -1243,7 +1232,7 @@
|
||||||
"parent_tax_article": "ƏDV-dən azad olunan əməliyyatlar"
|
"parent_tax_article": "ƏDV-dən azad olunan əməliyyatlar"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"article_name": "VM-nin 164.1.51-ci maddəsinə əsasən \"Tibbi sığorta haqqında\" Azərbaycan Respublikasının Qanununa uyğun olaraq icbari tibbi sığorta fondunun, habelə siyahısı müvafiq icra hakimiyyəti orqanının müəyyən etdiyi orqan (qurum) tərəfindən təsdiq edilən ictimai və sosial məqsədlər üçün yaradılmış fondların vəsaiti hesabına tibbi xidmətlərin göstərilməsi",
|
"article_name": "VM-nin 164.1.51-1-ci maddəsinə əsasən tədris bazası kimi fəaliyyət göstərən tibb müəssisələrində tibbi xidmətlərin göstərilməsi",
|
||||||
"declaration": "Declaration of value added tax",
|
"declaration": "Declaration of value added tax",
|
||||||
"docstatus": 0,
|
"docstatus": 0,
|
||||||
"doctype": "Tax Article",
|
"doctype": "Tax Article",
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,13 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
Normalize Tax Articles - fix Unicode normalization and curly quotes
|
Normalize Tax Articles - fix Unicode normalization, curly quotes, and NBSP
|
||||||
|
|
||||||
This script:
|
This script:
|
||||||
1. Replaces typographic (curly) quotes with straight quotes
|
1. Replaces non-breaking spaces (NBSP U+00A0) with regular spaces
|
||||||
2. Normalizes Unicode to NFC form (composed characters)
|
2. Replaces typographic (curly) quotes with straight quotes
|
||||||
|
3. Normalizes Unicode to NFC form (composed characters)
|
||||||
|
4. Fixes both `name` and `article_name` fields
|
||||||
|
|
||||||
Run after migration to ensure all Tax Article names match the code mapping.
|
Run after migration to ensure all Tax Article names match the code mapping.
|
||||||
"""
|
"""
|
||||||
|
|
@ -16,7 +18,8 @@ import unicodedata
|
||||||
|
|
||||||
def normalize_tax_articles():
|
def normalize_tax_articles():
|
||||||
"""
|
"""
|
||||||
Normalize all Tax Article article_name values:
|
Normalize all Tax Article name and article_name values:
|
||||||
|
- Replace non-breaking spaces (U+00A0) with regular spaces
|
||||||
- Replace typographic quotes (U+201C, U+201D) with straight quotes (")
|
- Replace typographic quotes (U+201C, U+201D) with straight quotes (")
|
||||||
- Normalize Unicode to NFC form
|
- Normalize Unicode to NFC form
|
||||||
"""
|
"""
|
||||||
|
|
@ -26,9 +29,38 @@ def normalize_tax_articles():
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Step 1: Replace curly quotes with straight quotes using HEX
|
# Step 1: Replace non-breaking spaces (NBSP) with regular spaces
|
||||||
# E2809C = left curly quote "
|
# C2A0 = UTF-8 for U+00A0 (NBSP)
|
||||||
# E2809D = right curly quote "
|
# 20 = regular space
|
||||||
|
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE `tabTax Article`
|
||||||
|
SET article_name = REPLACE(article_name, UNHEX('C2A0'), ' ')
|
||||||
|
WHERE HEX(article_name) LIKE '%C2A0%'
|
||||||
|
""")
|
||||||
|
nbsp_article_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0]
|
||||||
|
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE `tabTax Article`
|
||||||
|
SET name = REPLACE(name, UNHEX('C2A0'), ' ')
|
||||||
|
WHERE HEX(name) LIKE '%C2A0%'
|
||||||
|
""")
|
||||||
|
nbsp_name_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0]
|
||||||
|
|
||||||
|
# Also fix NBSP in Sales Invoice Item tax_article references
|
||||||
|
if frappe.db.table_exists("Sales Invoice Item"):
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE `tabSales Invoice Item`
|
||||||
|
SET tax_article = REPLACE(tax_article, UNHEX('C2A0'), ' ')
|
||||||
|
WHERE HEX(tax_article) LIKE '%C2A0%'
|
||||||
|
""")
|
||||||
|
nbsp_ref_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0]
|
||||||
|
else:
|
||||||
|
nbsp_ref_fixed = 0
|
||||||
|
|
||||||
|
# Step 2: Replace curly quotes with straight quotes using HEX
|
||||||
|
# E2809C = left curly quote \u201c
|
||||||
|
# E2809D = right curly quote \u201d
|
||||||
# 22 = straight quote "
|
# 22 = straight quote "
|
||||||
|
|
||||||
frappe.db.sql("""
|
frappe.db.sql("""
|
||||||
|
|
@ -43,7 +75,7 @@ def normalize_tax_articles():
|
||||||
|
|
||||||
quotes_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0]
|
quotes_fixed = frappe.db.sql("SELECT ROW_COUNT()")[0][0]
|
||||||
|
|
||||||
# Step 2: Normalize Unicode to NFC form
|
# Step 3: Normalize Unicode to NFC form
|
||||||
articles = frappe.db.sql("""
|
articles = frappe.db.sql("""
|
||||||
SELECT name, article_name
|
SELECT name, article_name
|
||||||
FROM `tabTax Article`
|
FROM `tabTax Article`
|
||||||
|
|
@ -51,8 +83,16 @@ def normalize_tax_articles():
|
||||||
""", as_dict=True)
|
""", as_dict=True)
|
||||||
|
|
||||||
nfc_fixed = 0
|
nfc_fixed = 0
|
||||||
|
encoding_errors = 0
|
||||||
|
|
||||||
for article in articles:
|
for article in articles:
|
||||||
|
try:
|
||||||
original = article['article_name']
|
original = article['article_name']
|
||||||
|
|
||||||
|
# Handle potential encoding issues
|
||||||
|
if isinstance(original, bytes):
|
||||||
|
original = original.decode('utf-8', errors='replace')
|
||||||
|
|
||||||
normalized = unicodedata.normalize('NFC', original)
|
normalized = unicodedata.normalize('NFC', original)
|
||||||
|
|
||||||
if original != normalized:
|
if original != normalized:
|
||||||
|
|
@ -63,10 +103,29 @@ def normalize_tax_articles():
|
||||||
""", (normalized, article['name']))
|
""", (normalized, article['name']))
|
||||||
nfc_fixed += 1
|
nfc_fixed += 1
|
||||||
|
|
||||||
|
except (UnicodeDecodeError, UnicodeEncodeError, TypeError) as e:
|
||||||
|
encoding_errors += 1
|
||||||
|
frappe.log_error(
|
||||||
|
f"Encoding error in Tax Article {article['name']}: {str(e)}",
|
||||||
|
"Tax Article Encoding Error"
|
||||||
|
)
|
||||||
|
|
||||||
frappe.db.commit()
|
frappe.db.commit()
|
||||||
|
|
||||||
if quotes_fixed > 0 or nfc_fixed > 0:
|
total_fixed = nbsp_article_fixed + nbsp_name_fixed + nbsp_ref_fixed + quotes_fixed + nfc_fixed
|
||||||
message = f"Tax Articles normalized: {quotes_fixed} curly quotes fixed, {nfc_fixed} NFC normalized"
|
if total_fixed > 0:
|
||||||
|
parts = []
|
||||||
|
if nbsp_article_fixed > 0 or nbsp_name_fixed > 0:
|
||||||
|
parts.append(f"{nbsp_article_fixed + nbsp_name_fixed} NBSP fixed")
|
||||||
|
if nbsp_ref_fixed > 0:
|
||||||
|
parts.append(f"{nbsp_ref_fixed} NBSP in references fixed")
|
||||||
|
if quotes_fixed > 0:
|
||||||
|
parts.append(f"{quotes_fixed} curly quotes fixed")
|
||||||
|
if nfc_fixed > 0:
|
||||||
|
parts.append(f"{nfc_fixed} NFC normalized")
|
||||||
|
if encoding_errors > 0:
|
||||||
|
parts.append(f"{encoding_errors} encoding errors")
|
||||||
|
message = f"Tax Articles normalized: {', '.join(parts)}"
|
||||||
print(message)
|
print(message)
|
||||||
frappe.log_error(message, "Tax Article Normalization")
|
frappe.log_error(message, "Tax Article Normalization")
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue