tools/ucd/normalization_test.mbt1///|2test "normalization generator rejects incomplete Quick_Check data" {3 try4 @ucd.emit_normalization(5 @ucd.parse_unicode_data(""),6 [],7 "00C0 ; NFD_QC ; N",8 )9 catch {10 Failure::Failure(message) =>11 assert_true(message.contains("Missing Quick_Check properties"))12 error => raise error13 } noraise {14 _ => fail("expected incomplete Quick_Check data to fail")15 }16}1718///|19test "normalization generator rejects Maybe for decomposition forms" {20 for property in ["NFD_QC", "NFKD_QC"] {21 try22 @ucd.emit_normalization(23 @ucd.parse_unicode_data(""),24 [],25 "0300 ; \{property} ; M",26 )27 catch {28 Failure::Failure(message) =>29 assert_true(message.contains("Invalid Quick_Check value"))30 error => raise error31 } noraise {32 _ => fail("expected invalid Maybe value to fail")33 }34 }35}3637///|38test "normalization generator rejects overlapping ranges including explicit Yes" {39 let input =40 #|0300..0302 ; NFC_QC ; Y41 #|0301 ; NFC_QC ; M42 try @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input) catch {43 Failure::Failure(message) =>44 assert_true(message.contains("Overlapping Quick_Check ranges"))45 error => raise error46 } noraise {47 _ => fail("expected overlapping ranges to fail")48 }49}5051///|52test "normalization generator rejects out of range code points" {53 try54 @ucd.emit_normalization(55 @ucd.parse_unicode_data(""),56 [],57 "110000 ; NFC_QC ; N",58 )59 catch {60 Failure::Failure(message) =>61 assert_true(message.contains("Invalid Quick_Check range"))62 error => raise error63 } noraise {64 _ => fail("expected out of range code point to fail")65 }66}6768///|69test "normalization generator accepts explicit defaults and unrelated properties" {70 let input =71 #|# @missing: 0000..10FFFF; NFC_QC; Yes72 #|0041 ; NFC_QC ; Y73 #|0041 ; NFKC_QC ; Y74 #|0041 ; NFD_QC ; Y75 #|0041 ; NFKD_QC ; Y76 #|0042 ; Full_Composition_Exclusion77 let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)78 // An all-default table has a BMP block and an implicit supplementary tail.79 assert_true(source.contains("if cp >= 65536"))80}8182///|83test "normalization generator validates the full supplementary range" {84 // Force a stored value at both supplementary endpoints, including the last85 // slot of the last leaf. emit_normalization validates every intervening index.86 let input =87 #|10000 ; NFC_QC ; N88 #|10FFFF ; NFC_QC ; N89 #|10FFFF ; NFKC_QC ; N90 #|10FFFF ; NFD_QC ; N91 #|10FFFF ; NFKD_QC ; N92 let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)93 assert_true(source.contains("if cp >= 1114112"))94}9596///|97test "normalization generator rejects surrogate scalar properties" {98 let input =99 #|D800 ; NFC_QC ; N100 #|D800 ; NFKC_QC ; N101 #|D800 ; NFD_QC ; N102 #|D800 ; NFKD_QC ; N103 try @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), input) catch {104 Failure::Failure(message) =>105 assert_true(message.contains("Surrogate code points must have default"))106 error => raise error107 } noraise {108 _ => fail("expected surrogate scalar properties to fail")109 }110}111112///|113test "normalization scan generator covers empty and full supplementary tables" {114 let defaults =115 #|0041 ; NFC_QC ; Y116 #|0041 ; NFKC_QC ; Y117 #|0041 ; NFD_QC ; Y118 #|0041 ; NFKD_QC ; Y119 let empty = @ucd.emit_normalization_scan(120 @ucd.parse_unicode_data(""),121 defaults,122 )123 assert_true(empty.contains("NormalizationTailLeadIndex : Int = 0"))124 let full =125 #|10000 ; NFC_QC ; M126 #|10FFFF ; NFC_QC ; N127 #|10FFFF ; NFKC_QC ; N128 #|10FFFF ; NFD_QC ; N129 #|10FFFF ; NFKD_QC ; N130 let source = @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), full)131 assert_true(source.contains("NormalizationTailLeadIndex : Int = 1024"))132}133134///|135test "normalization scan generator reduces properties to one byte" {136 let quick_check =137 #|0041 ; NFC_QC ; M138 #|0041 ; NFKC_QC ; Y139 #|0041 ; NFD_QC ; Y140 #|0041 ; NFKD_QC ; N141 let source = @ucd.emit_normalization_scan(142 @ucd.parse_unicode_data("0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;"),143 quick_check,144 )145 // NFC_QC=Maybe is bit 9 and NFKD_QC=No is bit 13 of the packed properties;146 // both move down seven bits, leaving bit 0 for a nonzero CCC.147 assert_true(source.contains(" 0,\n 68,\n 0,\n"))148 assert_true(source.contains(" 0,\n 1,\n 0,\n"))149 // Leading surrogates fill whole blocks with their marker.150 assert_true(source.contains(" 128,\n".repeat(32)))151 assert_true(152 source.contains("const NormalizationScanLeadSurrogate : Int = 128"),153 )154}155156///|157test "normalization scan generator gives flagged supplementary groups a slot" {158 let quick_check =159 #|0041 ; NFC_QC ; Y160 #|0041 ; NFKC_QC ; Y161 #|0041 ; NFD_QC ; Y162 #|10401 ; NFKD_QC ; N163 let source = @ucd.emit_normalization_scan(164 @ucd.parse_unicode_data(""),165 quick_check,166 )167 // Lead index 0 is clear and shares slot 0; lead index 1 gets slot 1.168 assert_true(169 source.contains("let normalization_lead_slots : Bytes = [\n 0,\n 1,\n]"),170 )171 // Two slots of 64 blocks; U+10401 is the second code point of slot 1.172 assert_true(173 source.contains(" 0,\n".repeat(64) + " 1,\n" + " 0,\n".repeat(63) + "]"),174 )175 assert_true(source.contains(" 0,\n".repeat(16) + " 0,\n 64,\n"))176}177
///|
test "normalization generator rejects incomplete Quick_Check data" {
try
@ucd.emit_normalization(
@ucd.parse_unicode_data(""),
[],
"00C0 ; NFD_QC ; N",
)
catch {
Failure::Failure(message) =>
assert_true(message.contains("Missing Quick_Check properties"))
error => raise error
} noraise {
_ => fail("expected incomplete Quick_Check data to fail")
}
}
///|
test "normalization generator rejects Maybe for decomposition forms" {
for property in ["NFD_QC", "NFKD_QC"] {
try
@ucd.emit_normalization(
@ucd.parse_unicode_data(""),
[],
"0300 ; \{property} ; M",
)
catch {
Failure::Failure(message) =>
assert_true(message.contains("Invalid Quick_Check value"))
error => raise error
} noraise {
_ => fail("expected invalid Maybe value to fail")
}
}
}
///|
test "normalization generator rejects overlapping ranges including explicit Yes" {
let input =
#|0300..0302 ; NFC_QC ; Y
#|0301 ; NFC_QC ; M
try @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input) catch {
Failure::Failure(message) =>
assert_true(message.contains("Overlapping Quick_Check ranges"))
error => raise error
} noraise {
_ => fail("expected overlapping ranges to fail")
}
}
///|
test "normalization generator rejects out of range code points" {
try
@ucd.emit_normalization(
@ucd.parse_unicode_data(""),
[],
"110000 ; NFC_QC ; N",
)
catch {
Failure::Failure(message) =>
assert_true(message.contains("Invalid Quick_Check range"))
error => raise error
} noraise {
_ => fail("expected out of range code point to fail")
}
}
///|
test "normalization generator accepts explicit defaults and unrelated properties" {
let input =
#|# @missing: 0000..10FFFF; NFC_QC; Yes
#|0041 ; NFC_QC ; Y
#|0041 ; NFKC_QC ; Y
#|0041 ; NFD_QC ; Y
#|0041 ; NFKD_QC ; Y
#|0042 ; Full_Composition_Exclusion
let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)
// An all-default table has a BMP block and an implicit supplementary tail.
assert_true(source.contains("if cp >= 65536"))
}
///|
test "normalization generator validates the full supplementary range" {
// Force a stored value at both supplementary endpoints, including the last
// slot of the last leaf. emit_normalization validates every intervening index.
let input =
#|10000 ; NFC_QC ; N
#|10FFFF ; NFC_QC ; N
#|10FFFF ; NFKC_QC ; N
#|10FFFF ; NFD_QC ; N
#|10FFFF ; NFKD_QC ; N
let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)
assert_true(source.contains("if cp >= 1114112"))
}
///|
test "normalization generator rejects surrogate scalar properties" {
let input =
#|D800 ; NFC_QC ; N
#|D800 ; NFKC_QC ; N
#|D800 ; NFD_QC ; N
#|D800 ; NFKD_QC ; N
try @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), input) catch {
Failure::Failure(message) =>
assert_true(message.contains("Surrogate code points must have default"))
error => raise error
} noraise {
_ => fail("expected surrogate scalar properties to fail")
}
}
///|
test "normalization scan generator covers empty and full supplementary tables" {
let defaults =
#|0041 ; NFC_QC ; Y
#|0041 ; NFKC_QC ; Y
#|0041 ; NFD_QC ; Y
#|0041 ; NFKD_QC ; Y
let empty = @ucd.emit_normalization_scan(
@ucd.parse_unicode_data(""),
defaults,
)
assert_true(empty.contains("NormalizationTailLeadIndex : Int = 0"))
let full =
#|10000 ; NFC_QC ; M
#|10FFFF ; NFC_QC ; N
#|10FFFF ; NFKC_QC ; N
#|10FFFF ; NFD_QC ; N
#|10FFFF ; NFKD_QC ; N
let source = @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), full)
assert_true(source.contains("NormalizationTailLeadIndex : Int = 1024"))
}
///|
test "normalization scan generator reduces properties to one byte" {
let quick_check =
#|0041 ; NFC_QC ; M
#|0041 ; NFKC_QC ; Y
#|0041 ; NFD_QC ; Y
#|0041 ; NFKD_QC ; N
let source = @ucd.emit_normalization_scan(
@ucd.parse_unicode_data("0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;"),
quick_check,
)
// NFC_QC=Maybe is bit 9 and NFKD_QC=No is bit 13 of the packed properties;
// both move down seven bits, leaving bit 0 for a nonzero CCC.
assert_true(source.contains(" 0,\n 68,\n 0,\n"))
assert_true(source.contains(" 0,\n 1,\n 0,\n"))
// Leading surrogates fill whole blocks with their marker.
assert_true(source.contains(" 128,\n".repeat(32)))
assert_true(
source.contains("const NormalizationScanLeadSurrogate : Int = 128"),
)
}
///|
test "normalization scan generator gives flagged supplementary groups a slot" {
let quick_check =
#|0041 ; NFC_QC ; Y
#|0041 ; NFKC_QC ; Y
#|0041 ; NFD_QC ; Y
#|10401 ; NFKD_QC ; N
let source = @ucd.emit_normalization_scan(
@ucd.parse_unicode_data(""),
quick_check,
)
// Lead index 0 is clear and shares slot 0; lead index 1 gets slot 1.
assert_true(
source.contains("let normalization_lead_slots : Bytes = [\n 0,\n 1,\n]"),
)
// Two slots of 64 blocks; U+10401 is the second code point of slot 1.
assert_true(
source.contains(" 0,\n".repeat(64) + " 1,\n" + " 0,\n".repeat(63) + "]"),
)
assert_true(source.contains(" 0,\n".repeat(16) + " 0,\n 64,\n"))
}