taxes_az/check_tax_articles_safe.py

"""
Tax Articles Verification Script (Safe version with encoding handling)
Run: bench --site [site-name] console
Then copy this code into the console
"""

import frappe
import unicodedata

def safe_str(text):
    """Safely convert text to string, handling encoding issues"""
    if not text:
        return ""
    try:
        # Try normal encoding
        return str(text)
    except (UnicodeDecodeError, UnicodeEncodeError):
        # If failed, use surrogateescape to handle bad bytes
        if isinstance(text, bytes):
            return text.decode('utf-8', errors='replace')
        return text.encode('utf-8', errors='replace').decode('utf-8')

print("=" * 100)
print("TAX ARTICLES VERIFICATION (SAFE MODE)")
print("=" * 100)

# 1. Check for curly quotes
print("\n1. CURLY QUOTES (should be 0):")
print("-" * 100)

try:
    curly_count = frappe.db.sql("""
        SELECT COUNT(*) as count
        FROM `tabTax Article`
        WHERE HEX(article_name) LIKE '%E2809C%'
           OR HEX(article_name) LIKE '%E2809D%'
    """)[0][0]

    if curly_count == 0:
        print("✓ No curly quotes found")
    else:
        print(f"✗ Found {curly_count} articles with curly quotes!")

        # Show problematic articles
        articles = frappe.db.sql("""
            SELECT name
            FROM `tabTax Article`
            WHERE HEX(article_name) LIKE '%E2809C%'
               OR HEX(article_name) LIKE '%E2809D%'
            LIMIT 5
        """, as_dict=True)

        for a in articles:
            print(f"  - {safe_str(a['name'][:70])}...")
except Exception as e:
    print(f"✗ Error checking curly quotes: {e}")

# 2. Check NFD/NFC normalization
print("\n2. NFD/NFC NORMALIZATION:")
print("-" * 100)

try:
    articles = frappe.db.sql("""
        SELECT name, article_name
        FROM `tabTax Article`
        WHERE article_name IS NOT NULL AND article_name != ''
    """, as_dict=True)

    nfd_issues = []
    encoding_errors = []

    for article in articles:
        try:
            original = article['article_name']
            # Try to normalize
            normalized = unicodedata.normalize('NFC', original)
            if len(original) != len(normalized):
                nfd_issues.append({
                    'name': article['name'],
                    'len_original': len(original),
                    'len_normalized': len(normalized)
                })
        except (UnicodeDecodeError, UnicodeEncodeError, TypeError) as e:
            encoding_errors.append({
                'name': article['name'],
                'error': str(e)
            })

    if not nfd_issues and not encoding_errors:
        print("✓ All articles are in NFC form")
    else:
        if nfd_issues:
            print(f"✗ Found {len(nfd_issues)} articles requiring normalization:")
            for issue in nfd_issues[:5]:
                print(f"  - {safe_str(issue['name'][:60])}... ({issue['len_original']} → {issue['len_normalized']})")

        if encoding_errors:
            print(f"\n⚠ Found {len(encoding_errors)} articles with encoding errors:")
            for issue in encoding_errors[:5]:
                print(f"  - {safe_str(issue['name'][:60])}...")
                print(f"    Error: {issue['error']}")
except Exception as e:
    print(f"✗ Error checking normalization: {e}")

# 3. Check matching with report code
print("\n3. MATCHING WITH REPORT CODE:")
print("-" * 100)

try:
    from taxes_az.taxes_az.report.tax_articles_revenue_report.tax_articles_revenue_report import get_tax_articles_mapping

    code_mapping = get_tax_articles_mapping()

    problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51']
    mismatches = []

    for key in problem_keys:
        articles = frappe.db.sql("""
            SELECT name, article_name
            FROM `tabTax Article`
            WHERE name LIKE %s
        """, (f'%{key}%',), as_dict=True)

        for article in articles:
            try:
                if article['article_name'] not in code_mapping:
                    mismatches.append({
                        'key': key,
                        'name': article['name'],
                        'article_name': safe_str(article['article_name'][:60])
                    })
            except (UnicodeDecodeError, UnicodeEncodeError):
                mismatches.append({
                    'key': key,
                    'name': article['name'],
                    'article_name': '[ENCODING ERROR]'
                })

    if not mismatches:
        print("✓ All problem articles found in code")
    else:
        print(f"✗ {len(mismatches)} articles NOT found in code mapping:")
        for m in mismatches:
            print(f"  - {m['key']}: {safe_str(m['name'][:60])}...")
except Exception as e:
    print(f"✗ Error checking code mapping: {e}")

# 4. Check data in Sales Invoice Items
print("\n4. DATA IN SALES INVOICE ITEMS:")
print("-" * 100)

try:
    problem_keys = ['164.1.34-1', '164.1.41-2', '164.1.42', '164.1.47', '164.1.51']

    for key in problem_keys:
        count = frappe.db.sql("""
            SELECT COUNT(*) as count
            FROM `tabSales Invoice Item` sii
            JOIN `tabSales Invoice` si ON sii.parent = si.name
            WHERE si.docstatus = 1
              AND sii.tax_article LIKE %s
        """, (f'%{key}%',))[0][0]

        status = "✓" if count > 0 else "✗"
        print(f"  {status} {key}: {count} records")
except Exception as e:
    print(f"✗ Error checking Sales Invoice Items: {e}")

# 5. Final summary
print("\n" + "=" * 100)
print("SUMMARY:")
print("=" * 100)

try:
    total_issues = curly_count + len(nfd_issues) + len(mismatches)

    if total_issues == 0:
        print("✓✓✓ ALL CHECKS PASSED! Tax Articles are OK.")
    else:
        print(f"⚠ Found {total_issues} issues:")
        if curly_count > 0:
            print(f"  - Curly quotes: {curly_count}")
        if nfd_issues:
            print(f"  - NFD normalization: {len(nfd_issues)}")
        if mismatches:
            print(f"  - Not in code: {len(mismatches)}")

        print("\nRun normalization:")
        print("from taxes_az.normalize_tax_articles import normalize_tax_articles")
        print("normalize_tax_articles()")
except Exception as e:
    print(f"Error in summary: {e}")

print("=" * 100)