normalization/normalization.mbt1///|2/// Unicode Normalization Forms (UAX #15)3///4/// This module implements the four Unicode normalization forms:5/// - NFD: Canonical Decomposition6/// - NFC: Canonical Decomposition followed by Canonical Composition7/// - NFKD: Compatibility Decomposition8/// - NFKC: Compatibility Decomposition followed by Canonical Composition910///|11/// Normalization form enumeration12#derive(Eq)13pub(all) enum NormalizationForm {14 NFD // Canonical Decomposition15 NFC // Canonical Decomposition + Composition16 NFKD // Compatibility Decomposition17 NFKC // Compatibility Decomposition + Composition18}1920///|21pub extend NormalizationForm with Eq::{not_equal, equal}2223///|24pub extend NormalizationForm with Show::{to_string, output}2526///|27pub impl Show for NormalizationForm with fn output(self, logger) {28 match self {29 NFD => logger.write_string("NFD")30 NFC => logger.write_string("NFC")31 NFKD => logger.write_string("NFKD")32 NFKC => logger.write_string("NFKC")33 }34}3536///|37/// Normalize a string to NFD (Canonical Decomposition)38///39/// Characters are decomposed to their canonical equivalents and40/// combining marks are reordered by Canonical Combining Class.41pub fn nfd(s : StringView) -> String {42 normalize_spans(s, 12, 1, compatibility=false, compose=false)43}4445///|46/// Normalize a string to NFC (Canonical Decomposition + Composition)47///48/// Characters are first decomposed canonically, reordered, then49/// recomposed using canonical composition rules.50/// This is the most commonly used normalization form.51pub fn nfc(s : StringView) -> String {52 normalize_spans(s, 8, 3, compatibility=false, compose=true)53}5455///|56/// Normalize a string to NFKD (Compatibility Decomposition)57///58/// Characters are decomposed to their compatibility equivalents59/// (including formatting variants like superscripts, fractions, etc.)60/// and combining marks are reordered.61pub fn nfkd(s : StringView) -> String {62 normalize_spans(s, 13, 1, compatibility=true, compose=false)63}6465///|66/// Normalize a string to NFKC (Compatibility Decomposition + Composition)67///68/// Characters are decomposed to compatibility equivalents, reordered,69/// then recomposed. Often used for identifier comparison.70pub fn nfkc(s : StringView) -> String {71 normalize_spans(s, 10, 3, compatibility=true, compose=true)72}7374///|75/// Normalize a string to the specified form76pub fn normalize(s : StringView, form : NormalizationForm) -> String {77 match form {78 NFD => nfd(s)79 NFC => nfc(s)80 NFKD => nfkd(s)81 NFKC => nfkc(s)82 }83}8485///|86/// Check if a string is in the specified normalization form87pub fn is_normalized(s : StringView, form : NormalizationForm) -> Bool {88 let result = match form {89 NFC =>90 process_spans(s, 8, 3, compatibility=false, compose=true, check_only=true)91 NFKC =>92 process_spans(s, 10, 3, compatibility=true, compose=true, check_only=true)93 NFD =>94 process_spans(95 s,96 12,97 1,98 compatibility=false,99 compose=false,100 check_only=true,101 )102 NFKD =>103 process_spans(104 s,105 13,106 1,107 compatibility=true,108 compose=false,109 check_only=true,110 )111 }112 result is Some(_)113}114
///|
/// Unicode Normalization Forms (UAX #15)
///
/// This module implements the four Unicode normalization forms:
/// - NFD: Canonical Decomposition
/// - NFC: Canonical Decomposition followed by Canonical Composition
/// - NFKD: Compatibility Decomposition
/// - NFKC: Compatibility Decomposition followed by Canonical Composition
///|
/// Normalization form enumeration
#derive(Eq)
pub(all) enum NormalizationForm {
NFD // Canonical Decomposition
NFC // Canonical Decomposition + Composition
NFKD // Compatibility Decomposition
NFKC // Compatibility Decomposition + Composition
}
///|
pub extend NormalizationForm with Eq::{not_equal, equal}
///|
pub extend NormalizationForm with Show::{to_string, output}
///|
pub impl Show for NormalizationForm with fn output(self, logger) {
match self {
NFD => logger.write_string("NFD")
NFC => logger.write_string("NFC")
NFKD => logger.write_string("NFKD")
NFKC => logger.write_string("NFKC")
}
}
///|
/// Normalize a string to NFD (Canonical Decomposition)
///
/// Characters are decomposed to their canonical equivalents and
/// combining marks are reordered by Canonical Combining Class.
pub fn nfd(s : StringView) -> String {
normalize_spans(s, 12, 1, compatibility=false, compose=false)
}
///|
/// Normalize a string to NFC (Canonical Decomposition + Composition)
///
/// Characters are first decomposed canonically, reordered, then
/// recomposed using canonical composition rules.
/// This is the most commonly used normalization form.
pub fn nfc(s : StringView) -> String {
normalize_spans(s, 8, 3, compatibility=false, compose=true)
}
///|
/// Normalize a string to NFKD (Compatibility Decomposition)
///
/// Characters are decomposed to their compatibility equivalents
/// (including formatting variants like superscripts, fractions, etc.)
/// and combining marks are reordered.
pub fn nfkd(s : StringView) -> String {
normalize_spans(s, 13, 1, compatibility=true, compose=false)
}
///|
/// Normalize a string to NFKC (Compatibility Decomposition + Composition)
///
/// Characters are decomposed to compatibility equivalents, reordered,
/// then recomposed. Often used for identifier comparison.
pub fn nfkc(s : StringView) -> String {
normalize_spans(s, 10, 3, compatibility=true, compose=true)
}
///|
/// Normalize a string to the specified form
pub fn normalize(s : StringView, form : NormalizationForm) -> String {
match form {
NFD => nfd(s)
NFC => nfc(s)
NFKD => nfkd(s)
NFKC => nfkc(s)
}
}
///|
/// Check if a string is in the specified normalization form
pub fn is_normalized(s : StringView, form : NormalizationForm) -> Bool {
let result = match form {
NFC =>
process_spans(s, 8, 3, compatibility=false, compose=true, check_only=true)
NFKC =>
process_spans(s, 10, 3, compatibility=true, compose=true, check_only=true)
NFD =>
process_spans(
s,
12,
1,
compatibility=false,
compose=false,
check_only=true,
)
NFKD =>
process_spans(
s,
13,
1,
compatibility=true,
compose=false,
check_only=true,
)
}
result is Some(_)
}