Unicode Properties and Categories: Classifying Every Character

import unicodedata

chars = ['A', 'a', '3', '中', '!', '+', '$', '😀', ' ', '\u0301']
for c in chars:
    cat = unicodedata.category(c)
    name = unicodedata.name(c, '(no name)')
    print(f'U+{ord(c):04X}  {cat}  {name}')

# U+0041  Lu  LATIN CAPITAL LETTER A
# U+0061  Ll  LATIN SMALL LETTER A
# U+0033  Nd  DIGIT THREE
# U+4E2D  Lo  CJK UNIFIED IDEOGRAPH-4E2D
# U+0021  Po  EXCLAMATION MARK
# U+002B  Sm  PLUS SIGN
# U+0024  Sc  DOLLAR SIGN
# U+1F600  So  GRINNING FACE
# U+0020  Zs  SPACE
# U+0301  Mn  COMBINING ACUTE ACCENT

import unicodedata

def count_by_category(text):
    """Count characters by their Unicode general category."""
    counts = {}
    for c in text:
        cat = unicodedata.category(c)
        counts[cat] = counts.get(cat, 0) + 1
    return counts

sample = "Hello, 世界! 42 + $3.14 = π"
print(count_by_category(sample))
# {'Lu': 1, 'Ll': 4, 'Po': 3, 'Zs': 5, 'Lo': 2, 'Nd': 3,
#  'Sm': 2, 'Sc': 1, 'Ll': 1}

# Extract only letters
letters_only = ''.join(c for c in text
                       if unicodedata.category(c).startswith('L'))

# Python 3.x doesn't expose Script directly from unicodedata
# Use the 'regex' package for script-aware matching
import regex

# Match any Arabic letter
arabic_pattern = regex.compile(r'\p{Script=Arabic}+')
text = "مرحبا بالعالم Hello مرحبا"
matches = arabic_pattern.findall(text)
print(matches)  # ['مرحبا', 'بالعالم', 'مرحبا']

# Match any Han (CJK) character
han_pattern = regex.compile(r'\p{Script=Han}+')
text2 = "Hello 世界 World 日本語"
print(han_pattern.findall(text2))  # ['世界', '日本語']

// Script matching
const arabicRE = /\p{Script=Arabic}+/gu;
const text = "مرحبا بالعالم Hello";
console.log(text.match(arabicRE));  // ['مرحبا', 'بالعالم']

// General category matching
const letterRE = /\p{Letter}+/gu;
const text2 = "Hello 42 世界 3.14";
console.log(text2.match(letterRE));  // ['Hello', '世界']

// Decimal digits in any script
const digitRE = /\p{Decimal_Number}+/gu;
const text3 = "42 ١٢٣ ৪৫৬";  // ASCII, Arabic-Indic, Bengali
console.log(text3.match(digitRE));  // ['42', '١٢٣', '৪৫৬']

// \p{Alphabetic} matches letters across all scripts
const alphaRE = /\p{Alphabetic}+/gu;
"café αβγ مرحبا 漢字".match(alphaRE);
// ['café', 'αβγ', 'مرحبا', '漢字']

import unicodedata

# Characters that can start a Python identifier
def can_start_identifier(c):
    cat = unicodedata.category(c)
    return cat in ('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') or c == '_'

# Python identifiers can use Unicode letters
# This is valid Python 3:
# π = 3.14159
# 変数 = "variable"
# café = "coffee shop"

import unicodedata

# unicodedata provides these as functions
print(unicodedata.category('A'))  # Lu (Uppercase Letter)
print(unicodedata.category('a'))  # Ll (Lowercase Letter)

# Check case properties
print('A'.isupper())   # True
print('a'.islower())   # True
print('中'.isupper())  # False
print('中'.islower())  # False — CJK has no case

# Case conversion works across scripts
print('ñoño'.upper())   # 'ÑOÑO'
print('ΑΘΗΝΑ'.lower())  # 'αθηνα' (Greek)
print('МОСКВА'.lower()) # 'москва' (Russian)

import unicodedata

# Not all whitespace is U+0020!
spaces = [
    '\u0020',   # SPACE
    '\u00A0',   # NO-BREAK SPACE
    '\u2003',   # EM SPACE
    '\u2009',   # THIN SPACE
    '\u200B',   # ZERO WIDTH SPACE (NOT white_space!)
    '\u3000',   # IDEOGRAPHIC SPACE (fullwidth)
]

for s in spaces:
    print(f'U+{ord(s):04X} {unicodedata.name(s)}: '
          f'category={unicodedata.category(s)}')

# In Python, str.split() and str.strip() use Unicode White_Space
# They handle all Unicode spaces, not just ASCII space
"hello\u2003world".split()  # ['hello', 'world']

import unicodedata

digits = ['5', '٥', '৫', '๕', '᪕', '⑤']
for d in digits:
    val = unicodedata.digit(d, None)
    nval = unicodedata.numeric(d, None)
    print(f'{d}  digit={val}  numeric={nval}  '
          f'name={unicodedata.name(d)}')

# 5  digit=5  numeric=5.0  DIGIT FIVE
# ٥  digit=5  numeric=5.0  ARABIC-INDIC DIGIT FIVE
# ৫  digit=5  numeric=5.0  BENGALI DIGIT FIVE
# ๕  digit=5  numeric=5.0  THAI DIGIT FIVE
# ᪕  digit=5  numeric=5.0  CHAM DIGIT FIVE
# ⑤  digit=None  numeric=5.0  CIRCLED DIGIT FIVE (not a decimal digit)

import regex

# Any Unicode letter
regex.findall(r'\p{L}+', "Hello 世界 مرحبا")
# ['Hello', '世界', 'مرحبا']

# Any decimal digit in any script
regex.findall(r'\p{Nd}+', "42 ٤٢ ৪২")
# ['42', '٤٢', '৪২']

# Negated: anything that is NOT a letter or digit
regex.findall(r'\P{Alnum}+', "hello, world! 42")
# [', ', '! ']

# Script-specific
regex.findall(r'\p{Script=Hiragana}+', "Hello はじめまして World")
# ['はじめまして']

# Currency symbols
regex.findall(r'\p{Sc}\d+', "Paid $42 and €15 and ¥1000")
# ['$42', '€15', '¥1000']

// Available in ES2018+ (Node.js 10+, modern browsers)
// Requires /u flag

// Match any letter
/\p{Letter}+/gu

// Match any number (all numeric categories)
/\p{Number}/gu

// Match emoji (using Emoji property)
/\p{Emoji}/gu

// Negate with \P
/\P{ASCII}/gu   // Any non-ASCII character

// Binary properties
/\p{White_Space}/gu
/\p{Uppercase}/gu
/\p{Lowercase}/gu
/\p{Alphabetic}/gu
/\p{Ideographic}/gu   // CJK ideographs

// Practical: validate that a string contains only letters and spaces
function isNameValid(name) {
    return /^[\p{Letter}\p{Space_Separator}]+$/u.test(name);
}

isNameValid("María García");    // true
isNameValid("田中太郎");          // true
isNameValid("User123");          // false (contains digits)

import unicodedata

# casefold() is Python's case-folding method (more aggressive than lower())
print('ß'.lower())      # 'ß' — German sharp s, lowercase stays ß
print('ß'.casefold())   # 'ss' — case-folded to two characters!

print('DŽ'.lower())      # 'dž' (titlecase → lowercase)
print('DŽ'.casefold())   # 'dž' (titlecase → lowercase casefold)

# For caseless comparison, use casefold()
def caseless_equal(s1, s2):
    return s1.casefold() == s2.casefold()

caseless_equal('CAFÉ', 'café')  # True
caseless_equal('straße', 'STRASSE')  # True (ß → ss)

// Find emoji in text
function extractEmoji(text) {
    return [...text.match(/\p{RGI_Emoji}/gv) || []];
}

// Note: /v flag (Unicode Sets mode) is required for RGI_Emoji
// Available in Node.js 20+, Chrome 112+, Safari 17+
extractEmoji("Hello 😀 World 🌍 café");  // ['😀', '🌍']