normalization/string_view_test.mbt1///|2test "normalization accepts bounded and nested string views" {3 let source = "😀!A\u{030A}ffié!😀"4 let view = source[2:source.length() - 2][1:5]5 inspect(@normalization.nfd(view), content="A\u{030A}ffie\u{0301}")6 inspect(@normalization.nfc(view), content="A\u{030A}ffié")7 inspect(@normalization.nfkd(view), content="A\u{030A}ffie\u{0301}")8 inspect(@normalization.nfkc(view), content="Åffié")9 // Neither a preceding starter nor a following mark belongs to the view.10 inspect(@normalization.nfc("A\u{030A}"[1:]), content="\u{030A}")11 inspect(@normalization.nfc("A\u{030A}"[:1]), content="A")12 assert_true(@normalization.is_normalized("A\u{030A}"[1:], @normalization.NFC))13 assert_true(@normalization.is_normalized("é"[1:1], @normalization.NFD))14 inspect(@normalization.nfd("é"[1:1]), content="")15}1617///|18test "normalization scanners respect view offsets and ends" {19 for prefix_length in 1..<20 {20 let prefix = "é".repeat(prefix_length)21 for gap_length in [0, 1, 15, 16, 17, 23, 24, 25, 64] {22 let gap = "x".repeat(gap_length)23 for24 sample in [25 "",26 "ASCII",27 "é",28 "e\u{0301}",29 "Affi",30 "x\u{0301}",31 "\u{0315}\u{0316}",32 "각",33 "😀",34 "\u{1D15E}",35 "\u{16D63}\u{16D68}",36 // Resume absolute scanning between suspect segments, then recover37 // the preceding starter inside a view with a nonzero start.38 "e\u{0301}\{gap}😀e\u{0301}",39 "x\u{0315}\u{0316}\{gap}\u{1D15E}",40 ] {41 let input = "\{gap}\{sample}\{gap}"42 let source = "\{prefix}\{input}é\u{0301}😀"43 let view = source[prefix.length():prefix.length() + input.length()]44 for45 form in [46 @normalization.NFD,47 @normalization.NFC,48 @normalization.NFKD,49 @normalization.NFKC,50 ] {51 assert_eq(52 @normalization.normalize(view, form),53 @normalization.normalize(input, form),54 )55 assert_eq(56 @normalization.is_normalized(view, form),57 @normalization.is_normalized(input, form),58 )59 }60 }61 }62 }63}64
///|
test "normalization accepts bounded and nested string views" {
let source = "😀!A\u{030A}ffié!😀"
let view = source[2:source.length() - 2][1:5]
inspect(@normalization.nfd(view), content="A\u{030A}ffie\u{0301}")
inspect(@normalization.nfc(view), content="A\u{030A}ffié")
inspect(@normalization.nfkd(view), content="A\u{030A}ffie\u{0301}")
inspect(@normalization.nfkc(view), content="Åffié")
// Neither a preceding starter nor a following mark belongs to the view.
inspect(@normalization.nfc("A\u{030A}"[1:]), content="\u{030A}")
inspect(@normalization.nfc("A\u{030A}"[:1]), content="A")
assert_true(@normalization.is_normalized("A\u{030A}"[1:], @normalization.NFC))
assert_true(@normalization.is_normalized("é"[1:1], @normalization.NFD))
inspect(@normalization.nfd("é"[1:1]), content="")
}
///|
test "normalization scanners respect view offsets and ends" {
for prefix_length in 1..<20 {
let prefix = "é".repeat(prefix_length)
for gap_length in [0, 1, 15, 16, 17, 23, 24, 25, 64] {
let gap = "x".repeat(gap_length)
for
sample in [
"",
"ASCII",
"é",
"e\u{0301}",
"Affi",
"x\u{0301}",
"\u{0315}\u{0316}",
"각",
"😀",
"\u{1D15E}",
"\u{16D63}\u{16D68}",
// Resume absolute scanning between suspect segments, then recover
// the preceding starter inside a view with a nonzero start.
"e\u{0301}\{gap}😀e\u{0301}",
"x\u{0315}\u{0316}\{gap}\u{1D15E}",
] {
let input = "\{gap}\{sample}\{gap}"
let source = "\{prefix}\{input}é\u{0301}😀"
let view = source[prefix.length():prefix.length() + input.length()]
for
form in [
@normalization.NFD,
@normalization.NFC,
@normalization.NFKD,
@normalization.NFKC,
] {
assert_eq(
@normalization.normalize(view, form),
@normalization.normalize(input, form),
)
assert_eq(
@normalization.is_normalized(view, form),
@normalization.is_normalized(input, form),
)
}
}
}
}
}