segmentation/grapheme_test.mbt1///|2/// Describe each cluster by its code points, so that the expected output3/// stays readable for combining marks, controls, and emoji sequences.4fn cluster_code_points(text : StringView) -> Array[Array[String]] {5 @segmentation.graphemes(text)6 .iter()7 .map(cluster => {8 cluster9 .iter()10 .map(character => "U+" + character.to_int().to_string(radix=16).to_upper())11 .collect()12 })13 .collect()14}1516///|17test "empty text has no clusters" {18 debug_inspect(19 cluster_code_points(""),20 content=(21 #|[]22 ),23 )24}2526///|27test "ASCII breaks between every character" {28 debug_inspect(29 cluster_code_points("ab 1"),30 content=(31 #|[["U+61"], ["U+62"], ["U+20"], ["U+31"]]32 ),33 )34}3536///|37test "CR LF is one cluster and controls stand alone" {38 debug_inspect(39 cluster_code_points("a\r\nb\n\r\u{0}\u{301}"),40 content=(41 #|[42 #| ["U+61"],43 #| ["U+D", "U+A"],44 #| ["U+62"],45 #| ["U+A"],46 #| ["U+D"],47 #| ["U+0"],48 #| ["U+301"],49 #|]50 ),51 )52}5354///|55test "combining marks and spacing marks extend the base" {56 debug_inspect(57 cluster_code_points("e\u{301}\u{323}o\u{308}\u{0915}\u{093E}"),58 content=(59 #|[["U+65", "U+301", "U+323"], ["U+6F", "U+308"], ["U+915", "U+93E"]]60 ),61 )62}6364///|65test "prepend characters attach to the following character" {66 debug_inspect(67 cluster_code_points("\u{0600}1\u{0600}\n"),68 content=(69 #|[["U+600", "U+31"], ["U+600"], ["U+A"]]70 ),71 )72}7374///|75test "Hangul syllable sequences" {76 // L V T jamo, a precomposed LV syllable followed by T, and LVT followed77 // by V, which starts a new cluster.78 debug_inspect(79 cluster_code_points(80 "\u{1100}\u{1161}\u{11A8}\u{AC00}\u{11A8}\u{AC01}\u{1161}",81 ),82 content=(83 #|[84 #| ["U+1100", "U+1161", "U+11A8"],85 #| ["U+AC00", "U+11A8"],86 #| ["U+AC01"],87 #| ["U+1161"],88 #|]89 ),90 )91}9293///|94test "regional indicators pair up from the start of a run" {95 debug_inspect(96 cluster_code_points("πΊπ³π¨π³π―aπ―π΅"),97 content=(98 #|[99 #| ["U+1F1FA", "U+1F1F3"],100 #| ["U+1F1E8", "U+1F1F3"],101 #| ["U+1F1EF"],102 #| ["U+61"],103 #| ["U+1F1EF", "U+1F1F5"],104 #|]105 ),106 )107}108109///|110test "emoji modifier and ZWJ sequences" {111 debug_inspect(112 cluster_code_points("ππ½π¨βπ©βπ§β€οΈβπ₯"),113 content=(114 #|[115 #| ["U+1F44D", "U+1F3FD"],116 #| ["U+1F468", "U+200D", "U+1F469", "U+200D", "U+1F467"],117 #| ["U+2764", "U+FE0F", "U+200D", "U+1F525"],118 #|]119 ),120 )121}122123///|124test "ZWJ joins only after an extended pictographic" {125 debug_inspect(126 cluster_code_points("a\u{200D}π§π¨\u{200D}a"),127 content=(128 #|[["U+61", "U+200D"], ["U+1F467"], ["U+1F468", "U+200D"], ["U+61"]]129 ),130 )131}132133///|134test "Indic conjuncts stay together across a linker" {135 // KA VIRAMA SSA is one conjunct; without the virama they are separate.136 debug_inspect(137 cluster_code_points("\u{0915}\u{094D}\u{0937}\u{0915}\u{0937}"),138 content=(139 #|[["U+915", "U+94D", "U+937"], ["U+915"], ["U+937"]]140 ),141 )142 // Extenders around the linker, including ZWJ, keep the conjunct open.143 debug_inspect(144 cluster_code_points("\u{0915}\u{093C}\u{094D}\u{200D}\u{0937}"),145 content=(146 #|[["U+915", "U+93C", "U+94D", "U+200D", "U+937"]]147 ),148 )149}150151///|152test "a linker joins the next consonant without a leading consonant" {153 // Since Unicode 18, GB9c no longer requires a consonant before the linker.154 debug_inspect(155 cluster_code_points("a\u{094D}\u{0924}"),156 content=(157 #|[["U+61", "U+94D", "U+924"]]158 ),159 )160 // VEDIC SIGN JIHVAMULIYA is a linker that is not Extend, so it starts the161 // cluster itself.162 debug_inspect(163 cluster_code_points("a\u{1CF5}\u{0915}\u{1CF5}a"),164 content=(165 #|[["U+61"], ["U+1CF5", "U+915"], ["U+1CF5"], ["U+61"]]166 ),167 )168}169170///|171test "clusters are views into the input" {172 let text = "xe\u{301}πΊπ³y"173 let offsets = @segmentation.graphemes(text[1:])174 .iter()175 .map(cluster => (cluster.start_offset(), cluster.length()))176 .collect()177 debug_inspect(178 offsets,179 content=(180 #|[(1, 2), (3, 4), (7, 1)]181 ),182 )183}184185///|186test "unpaired surrogates behave like ordinary characters" {187 // Slicing by code unit offsets can split a surrogate pair. Each half is a188 // base character of its own, so the low half takes the combining mark.189 let text = "aπ\u{301}"190 debug_inspect(191 @segmentation.graphemes(text.unsafe_view(start=0, end=2))192 .iter()193 .map(cluster => cluster.length())194 .collect(),195 content=(196 #|[1, 1]197 ),198 )199 debug_inspect(200 @segmentation.graphemes(text.unsafe_view(start=2, end=text.length()))201 .iter()202 .map(cluster => cluster.length())203 .collect(),204 content=(205 #|[2]206 ),207 )208}209210///|211test "next returns an empty view at the end" {212 let text = "xe\u{301}y"213 let clusters = @segmentation.graphemes(text[1:])214 let first = clusters.next()215 debug_inspect(216 (first.to_owned(), first.start_offset()),217 content=(218 #|("eΜ", 1)219 ),220 )221 let second = clusters.next()222 debug_inspect(223 (second.to_owned(), second.start_offset()),224 content=(225 #|("y", 3)226 ),227 )228 // The empty view sits at the end of the text, and `next` keeps returning it.229 let end = clusters.next()230 debug_inspect(231 (end.is_empty(), end.start_offset()),232 content=(233 #|(true, 4)234 ),235 )236 debug_inspect(237 clusters.next().is_empty(),238 content=(239 #|true240 ),241 )242 inspect(243 @segmentation.graphemes("").next().is_empty(),244 content=(245 #|true246 ),247 )248}249250///|251test "iter and next share the cursor" {252 let clusters = @segmentation.graphemes("abcd")253 inspect(254 clusters.next(),255 content=(256 #|a257 ),258 )259 let iter = clusters.iter()260 debug_inspect(261 iter.next().map(cluster => cluster.to_owned()),262 content=(263 #|Some("b")264 ),265 )266 inspect(267 clusters.next(),268 content=(269 #|c270 ),271 )272 debug_inspect(273 iter.map(cluster => cluster.to_owned()).collect(),274 content=(275 #|["d"]276 ),277 )278 inspect(279 clusters.next().is_empty(),280 content=(281 #|true282 ),283 )284}285286///|287test "for .. in iterates the clusters" {288 let lengths = []289 for cluster in @segmentation.graphemes("e\u{301}πΊπ³x") {290 lengths.push(cluster.length())291 }292 debug_inspect(293 lengths,294 content=(295 #|[2, 4, 1]296 ),297 )298}299
///|
/// Describe each cluster by its code points, so that the expected output
/// stays readable for combining marks, controls, and emoji sequences.
fn cluster_code_points(text : StringView) -> Array[Array[String]] {
@segmentation.graphemes(text)
.iter()
.map(cluster => {
cluster
.iter()
.map(character => "U+" + character.to_int().to_string(radix=16).to_upper())
.collect()
})
.collect()
}
///|
test "empty text has no clusters" {
debug_inspect(
cluster_code_points(""),
content=(
#|[]
),
)
}
///|
test "ASCII breaks between every character" {
debug_inspect(
cluster_code_points("ab 1"),
content=(
#|[["U+61"], ["U+62"], ["U+20"], ["U+31"]]
),
)
}
///|
test "CR LF is one cluster and controls stand alone" {
debug_inspect(
cluster_code_points("a\r\nb\n\r\u{0}\u{301}"),
content=(
#|[
#| ["U+61"],
#| ["U+D", "U+A"],
#| ["U+62"],
#| ["U+A"],
#| ["U+D"],
#| ["U+0"],
#| ["U+301"],
#|]
),
)
}
///|
test "combining marks and spacing marks extend the base" {
debug_inspect(
cluster_code_points("e\u{301}\u{323}o\u{308}\u{0915}\u{093E}"),
content=(
#|[["U+65", "U+301", "U+323"], ["U+6F", "U+308"], ["U+915", "U+93E"]]
),
)
}
///|
test "prepend characters attach to the following character" {
debug_inspect(
cluster_code_points("\u{0600}1\u{0600}\n"),
content=(
#|[["U+600", "U+31"], ["U+600"], ["U+A"]]
),
)
}
///|
test "Hangul syllable sequences" {
// L V T jamo, a precomposed LV syllable followed by T, and LVT followed
// by V, which starts a new cluster.
debug_inspect(
cluster_code_points(
"\u{1100}\u{1161}\u{11A8}\u{AC00}\u{11A8}\u{AC01}\u{1161}",
),
content=(
#|[
#| ["U+1100", "U+1161", "U+11A8"],
#| ["U+AC00", "U+11A8"],
#| ["U+AC01"],
#| ["U+1161"],
#|]
),
)
}
///|
test "regional indicators pair up from the start of a run" {
debug_inspect(
cluster_code_points("πΊπ³π¨π³π―aπ―π΅"),
content=(
#|[
#| ["U+1F1FA", "U+1F1F3"],
#| ["U+1F1E8", "U+1F1F3"],
#| ["U+1F1EF"],
#| ["U+61"],
#| ["U+1F1EF", "U+1F1F5"],
#|]
),
)
}
///|
test "emoji modifier and ZWJ sequences" {
debug_inspect(
cluster_code_points("ππ½π¨βπ©βπ§β€οΈβπ₯"),
content=(
#|[
#| ["U+1F44D", "U+1F3FD"],
#| ["U+1F468", "U+200D", "U+1F469", "U+200D", "U+1F467"],
#| ["U+2764", "U+FE0F", "U+200D", "U+1F525"],
#|]
),
)
}
///|
test "ZWJ joins only after an extended pictographic" {
debug_inspect(
cluster_code_points("a\u{200D}π§π¨\u{200D}a"),
content=(
#|[["U+61", "U+200D"], ["U+1F467"], ["U+1F468", "U+200D"], ["U+61"]]
),
)
}
///|
test "Indic conjuncts stay together across a linker" {
// KA VIRAMA SSA is one conjunct; without the virama they are separate.
debug_inspect(
cluster_code_points("\u{0915}\u{094D}\u{0937}\u{0915}\u{0937}"),
content=(
#|[["U+915", "U+94D", "U+937"], ["U+915"], ["U+937"]]
),
)
// Extenders around the linker, including ZWJ, keep the conjunct open.
debug_inspect(
cluster_code_points("\u{0915}\u{093C}\u{094D}\u{200D}\u{0937}"),
content=(
#|[["U+915", "U+93C", "U+94D", "U+200D", "U+937"]]
),
)
}
///|
test "a linker joins the next consonant without a leading consonant" {
// Since Unicode 18, GB9c no longer requires a consonant before the linker.
debug_inspect(
cluster_code_points("a\u{094D}\u{0924}"),
content=(
#|[["U+61", "U+94D", "U+924"]]
),
)
// VEDIC SIGN JIHVAMULIYA is a linker that is not Extend, so it starts the
// cluster itself.
debug_inspect(
cluster_code_points("a\u{1CF5}\u{0915}\u{1CF5}a"),
content=(
#|[["U+61"], ["U+1CF5", "U+915"], ["U+1CF5"], ["U+61"]]
),
)
}
///|
test "clusters are views into the input" {
let text = "xe\u{301}πΊπ³y"
let offsets = @segmentation.graphemes(text[1:])
.iter()
.map(cluster => (cluster.start_offset(), cluster.length()))
.collect()
debug_inspect(
offsets,
content=(
#|[(1, 2), (3, 4), (7, 1)]
),
)
}
///|
test "unpaired surrogates behave like ordinary characters" {
// Slicing by code unit offsets can split a surrogate pair. Each half is a
// base character of its own, so the low half takes the combining mark.
let text = "aπ\u{301}"
debug_inspect(
@segmentation.graphemes(text.unsafe_view(start=0, end=2))
.iter()
.map(cluster => cluster.length())
.collect(),
content=(
#|[1, 1]
),
)
debug_inspect(
@segmentation.graphemes(text.unsafe_view(start=2, end=text.length()))
.iter()
.map(cluster => cluster.length())
.collect(),
content=(
#|[2]
),
)
}
///|
test "next returns an empty view at the end" {
let text = "xe\u{301}y"
let clusters = @segmentation.graphemes(text[1:])
let first = clusters.next()
debug_inspect(
(first.to_owned(), first.start_offset()),
content=(
#|("eΜ", 1)
),
)
let second = clusters.next()
debug_inspect(
(second.to_owned(), second.start_offset()),
content=(
#|("y", 3)
),
)
// The empty view sits at the end of the text, and `next` keeps returning it.
let end = clusters.next()
debug_inspect(
(end.is_empty(), end.start_offset()),
content=(
#|(true, 4)
),
)
debug_inspect(
clusters.next().is_empty(),
content=(
#|true
),
)
inspect(
@segmentation.graphemes("").next().is_empty(),
content=(
#|true
),
)
}
///|
test "iter and next share the cursor" {
let clusters = @segmentation.graphemes("abcd")
inspect(
clusters.next(),
content=(
#|a
),
)
let iter = clusters.iter()
debug_inspect(
iter.next().map(cluster => cluster.to_owned()),
content=(
#|Some("b")
),
)
inspect(
clusters.next(),
content=(
#|c
),
)
debug_inspect(
iter.map(cluster => cluster.to_owned()).collect(),
content=(
#|["d"]
),
)
inspect(
clusters.next().is_empty(),
content=(
#|true
),
)
}
///|
test "for .. in iterates the clusters" {
let lengths = []
for cluster in @segmentation.graphemes("e\u{301}πΊπ³x") {
lengths.push(cluster.length())
}
debug_inspect(
lengths,
content=(
#|[2, 4, 1]
),
)
}