unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

normalization/normalization.mbt
1///|2/// Unicode Normalization Forms (UAX #15)3///4/// This module implements the four Unicode normalization forms:5/// - NFD: Canonical Decomposition6/// - NFC: Canonical Decomposition followed by Canonical Composition7/// - NFKD: Compatibility Decomposition8/// - NFKC: Compatibility Decomposition followed by Canonical Composition910///|11/// Normalization form enumeration12#derive(Eq)13pub(all) enum NormalizationForm {14  NFD // Canonical Decomposition15  NFC // Canonical Decomposition + Composition16  NFKD // Compatibility Decomposition17  NFKC // Compatibility Decomposition + Composition18}1920///|21pub extend NormalizationForm with Eq::{not_equal, equal}2223///|24pub extend NormalizationForm with Show::{to_string, output}2526///|27pub impl Show for NormalizationForm with fn output(self, logger) {28  match self {29    NFD => logger.write_string("NFD")30    NFC => logger.write_string("NFC")31    NFKD => logger.write_string("NFKD")32    NFKC => logger.write_string("NFKC")33  }34}3536///|37/// Normalize a string to NFD (Canonical Decomposition)38///39/// Characters are decomposed to their canonical equivalents and40/// combining marks are reordered by Canonical Combining Class.41pub fn nfd(s : StringView) -> String {42  normalize_spans(s, 12, 1, compatibility=false, compose=false)43}4445///|46/// Normalize a string to NFC (Canonical Decomposition + Composition)47///48/// Characters are first decomposed canonically, reordered, then49/// recomposed using canonical composition rules.50/// This is the most commonly used normalization form.51pub fn nfc(s : StringView) -> String {52  normalize_spans(s, 8, 3, compatibility=false, compose=true)53}5455///|56/// Normalize a string to NFKD (Compatibility Decomposition)57///58/// Characters are decomposed to their compatibility equivalents59/// (including formatting variants like superscripts, fractions, etc.)60/// and combining marks are reordered.61pub fn nfkd(s : StringView) -> String {62  normalize_spans(s, 13, 1, compatibility=true, compose=false)63}6465///|66/// Normalize a string to NFKC (Compatibility Decomposition + Composition)67///68/// Characters are decomposed to compatibility equivalents, reordered,69/// then recomposed. Often used for identifier comparison.70pub fn nfkc(s : StringView) -> String {71  normalize_spans(s, 10, 3, compatibility=true, compose=true)72}7374///|75/// Normalize a string to the specified form76pub fn normalize(s : StringView, form : NormalizationForm) -> String {77  match form {78    NFD => nfd(s)79    NFC => nfc(s)80    NFKD => nfkd(s)81    NFKC => nfkc(s)82  }83}8485///|86/// Check if a string is in the specified normalization form87pub fn is_normalized(s : StringView, form : NormalizationForm) -> Bool {88  let result = match form {89    NFC =>90      process_spans(s, 8, 3, compatibility=false, compose=true, check_only=true)91    NFKC =>92      process_spans(s, 10, 3, compatibility=true, compose=true, check_only=true)93    NFD =>94      process_spans(95        s,96        12,97        1,98        compatibility=false,99        compose=false,100        check_only=true,101      )102    NFKD =>103      process_spans(104        s,105        13,106        1,107        compatibility=true,108        compose=false,109        check_only=true,110      )111  }112  result is Some(_)113}114