segmentation/grapheme_line_break_test.mbt1///|2fn grapheme_breaks(3 text : StringView,4) -> Array[(String, @segmentation.BreakOpportunity)] {5 let cursor = @segmentation.graphemes(text)6 let results = []7 for cluster in cursor {8 results.push((cluster.to_owned(), cursor.line_break()))9 }10 results11}1213///|14test "grapheme line break kinds" {15 debug_inspect(16 grapheme_breaks("ab cd\n"),17 content=(18 #|[19 #| ("a", Prohibited),20 #| ("b", Prohibited),21 #| (" ", Allowed),22 #| ("c", Prohibited),23 #| ("d", Prohibited),24 #| ("\n", Mandatory),25 #|]26 ),27 )28 debug_inspect(29 grapheme_breaks("中文"),30 content="[(\"中\", Allowed), (\"文\", Mandatory)]",31 )32 debug_inspect(33 grapheme_breaks("A\r\nB"),34 content=(35 #|[("A", Prohibited), ("\r\n", Mandatory), ("B", Mandatory)]36 ),37 )38 for39 newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {40 @debug.assert_eq(grapheme_breaks("A" + newline + "B"), [41 ("A", @segmentation.Prohibited),42 (newline, @segmentation.Mandatory),43 ("B", @segmentation.Mandatory),44 ])45 }46}4748///|49test "grapheme line break query does not advance and survives exhaustion" {50 let empty = @segmentation.graphemes("")51 debug_inspect(empty.line_break(), content="Prohibited")52 assert_true(empty.next().is_empty())53 debug_inspect(empty.line_break(), content="Prohibited")54 let cursor = @segmentation.graphemes("a b")55 debug_inspect(cursor.line_break(), content="Prohibited")56 inspect(cursor.next(), content="a")57 debug_inspect(cursor.line_break(), content="Prohibited")58 debug_inspect(cursor.line_break(), content="Prohibited")59 let iterator = cursor.iter()60 debug_inspect(61 iterator.next(),62 content=(63 #|Some(<StringView: " ">)64 ),65 )66 debug_inspect(cursor.line_break(), content="Allowed")67 debug_inspect(cursor.line_break(), content="Allowed")68 inspect(cursor.next(), content="b")69 debug_inspect(cursor.line_break(), content="Mandatory")70 assert_true(iterator.next() is None)71 assert_true(cursor.next().is_empty())72 assert_true(cursor.next().is_empty())73 debug_inspect(cursor.line_break(), content="Mandatory")74}7576///|77test "line break queries can start late or skip clusters" {78 let cursor = @segmentation.graphemes("a b c")79 inspect(cursor.next(), content="a")80 inspect(cursor.next(), content=" ")81 debug_inspect(cursor.line_break(), content="Allowed")82 inspect(cursor.next(), content="b")83 inspect(cursor.next(), content=" ")84 debug_inspect(cursor.line_break(), content="Allowed")85 inspect(cursor.next(), content="c")86 debug_inspect(cursor.line_break(), content="Mandatory")87 let late = @segmentation.graphemes("x")88 assert_eq(late.iter().count(), 1)89 assert_true(late.next().is_empty())90 debug_inspect(late.line_break(), content="Mandatory")91}9293///|94test "grapheme line breaking preserves whole clusters and default rule context" {95 // UAX #14 allows a break after SPACE inside the first UAX #29 cluster.96 // That opportunity is suppressed, not moved to the cluster's end.97 debug_inspect(98 grapheme_breaks(" \u{0308}a"),99 content="[(\" ̈\", Prohibited), (\"a\", Mandatory)]",100 )101 debug_inspect(102 grapheme_breaks("e\u{0301}x"),103 content="[(\"é\", Prohibited), (\"x\", Mandatory)]",104 )105 debug_inspect(106 grapheme_breaks("🇺🇳🇨🇳"),107 content="[(\"🇺🇳\", Allowed), (\"🇨🇳\", Mandatory)]",108 )109 debug_inspect(110 grapheme_breaks("👨👩👧x"),111 content="[(\"👨👩👧\", Allowed), (\"x\", Mandatory)]",112 )113}114115///|116test "grapheme line breaking uses independent UTF-16 views" {117 let source = "x😀 中z"118 let text = source.unsafe_view(start=1, end=5)119 debug_inspect(120 grapheme_breaks(text),121 content="[(\"😀\", Prohibited), (\" \", Allowed), (\"中\", Mandatory)]",122 )123 let quoted = "中’中"124 debug_inspect(125 grapheme_breaks(quoted.unsafe_view(start=1, end=quoted.length())),126 content="[(\"’\", Prohibited), (\"中\", Mandatory)]",127 )128 let pair = "😀"129 for130 half in [131 pair.unsafe_view(start=0, end=1),132 pair.unsafe_view(start=1, end=pair.length()),133 ] {134 let cursor = @segmentation.graphemes(half)135 let cluster = cursor.next()136 assert_eq(cluster.start_offset(), half.start_offset())137 assert_eq(cluster.length(), 1)138 debug_inspect(cursor.line_break(), content="Mandatory")139 assert_true(cursor.next().is_empty())140 }141}142
///|
fn grapheme_breaks(
text : StringView,
) -> Array[(String, @segmentation.BreakOpportunity)] {
let cursor = @segmentation.graphemes(text)
let results = []
for cluster in cursor {
results.push((cluster.to_owned(), cursor.line_break()))
}
results
}
///|
test "grapheme line break kinds" {
debug_inspect(
grapheme_breaks("ab cd\n"),
content=(
#|[
#| ("a", Prohibited),
#| ("b", Prohibited),
#| (" ", Allowed),
#| ("c", Prohibited),
#| ("d", Prohibited),
#| ("\n", Mandatory),
#|]
),
)
debug_inspect(
grapheme_breaks("中文"),
content="[(\"中\", Allowed), (\"文\", Mandatory)]",
)
debug_inspect(
grapheme_breaks("A\r\nB"),
content=(
#|[("A", Prohibited), ("\r\n", Mandatory), ("B", Mandatory)]
),
)
for
newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {
@debug.assert_eq(grapheme_breaks("A" + newline + "B"), [
("A", @segmentation.Prohibited),
(newline, @segmentation.Mandatory),
("B", @segmentation.Mandatory),
])
}
}
///|
test "grapheme line break query does not advance and survives exhaustion" {
let empty = @segmentation.graphemes("")
debug_inspect(empty.line_break(), content="Prohibited")
assert_true(empty.next().is_empty())
debug_inspect(empty.line_break(), content="Prohibited")
let cursor = @segmentation.graphemes("a b")
debug_inspect(cursor.line_break(), content="Prohibited")
inspect(cursor.next(), content="a")
debug_inspect(cursor.line_break(), content="Prohibited")
debug_inspect(cursor.line_break(), content="Prohibited")
let iterator = cursor.iter()
debug_inspect(
iterator.next(),
content=(
#|Some(<StringView: " ">)
),
)
debug_inspect(cursor.line_break(), content="Allowed")
debug_inspect(cursor.line_break(), content="Allowed")
inspect(cursor.next(), content="b")
debug_inspect(cursor.line_break(), content="Mandatory")
assert_true(iterator.next() is None)
assert_true(cursor.next().is_empty())
assert_true(cursor.next().is_empty())
debug_inspect(cursor.line_break(), content="Mandatory")
}
///|
test "line break queries can start late or skip clusters" {
let cursor = @segmentation.graphemes("a b c")
inspect(cursor.next(), content="a")
inspect(cursor.next(), content=" ")
debug_inspect(cursor.line_break(), content="Allowed")
inspect(cursor.next(), content="b")
inspect(cursor.next(), content=" ")
debug_inspect(cursor.line_break(), content="Allowed")
inspect(cursor.next(), content="c")
debug_inspect(cursor.line_break(), content="Mandatory")
let late = @segmentation.graphemes("x")
assert_eq(late.iter().count(), 1)
assert_true(late.next().is_empty())
debug_inspect(late.line_break(), content="Mandatory")
}
///|
test "grapheme line breaking preserves whole clusters and default rule context" {
// UAX #14 allows a break after SPACE inside the first UAX #29 cluster.
// That opportunity is suppressed, not moved to the cluster's end.
debug_inspect(
grapheme_breaks(" \u{0308}a"),
content="[(\" ̈\", Prohibited), (\"a\", Mandatory)]",
)
debug_inspect(
grapheme_breaks("e\u{0301}x"),
content="[(\"é\", Prohibited), (\"x\", Mandatory)]",
)
debug_inspect(
grapheme_breaks("🇺🇳🇨🇳"),
content="[(\"🇺🇳\", Allowed), (\"🇨🇳\", Mandatory)]",
)
debug_inspect(
grapheme_breaks("👨👩👧x"),
content="[(\"👨👩👧\", Allowed), (\"x\", Mandatory)]",
)
}
///|
test "grapheme line breaking uses independent UTF-16 views" {
let source = "x😀 中z"
let text = source.unsafe_view(start=1, end=5)
debug_inspect(
grapheme_breaks(text),
content="[(\"😀\", Prohibited), (\" \", Allowed), (\"中\", Mandatory)]",
)
let quoted = "中’中"
debug_inspect(
grapheme_breaks(quoted.unsafe_view(start=1, end=quoted.length())),
content="[(\"’\", Prohibited), (\"中\", Mandatory)]",
)
let pair = "😀"
for
half in [
pair.unsafe_view(start=0, end=1),
pair.unsafe_view(start=1, end=pair.length()),
] {
let cursor = @segmentation.graphemes(half)
let cluster = cursor.next()
assert_eq(cluster.start_offset(), half.start_offset())
assert_eq(cluster.length(), 1)
debug_inspect(cursor.line_break(), content="Mandatory")
assert_true(cursor.next().is_empty())
}
}