segmentation/line_break_test.mbt1///|2fn line_break_points(3 text : StringView,4) -> Array[(Int32, @segmentation.BreakOpportunity)] {5 let cursor = @segmentation.line_breaks(text)6 let points = []7 for segment in cursor {8 points.push(9 (10 segment.start_offset() - text.start_offset() + segment.length(),11 cursor.opportunity(),12 ),13 )14 }15 points16}1718///|19test "allowed, mandatory, and end of text" {20 debug_inspect(21 line_break_points("Hello world!"),22 content="[(6, Allowed), (12, Mandatory)]",23 )24 debug_inspect(25 line_break_points("中文"),26 content="[(1, Allowed), (2, Mandatory)]",27 )28 debug_inspect(29 line_break_points("A\r\nB"),30 content="[(3, Mandatory), (4, Mandatory)]",31 )32 for33 newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {34 @debug.assert_eq(line_break_points("A" + newline + "B"), [35 (2, @segmentation.Mandatory),36 (3, @segmentation.Mandatory),37 ])38 }39 debug_inspect(line_break_points("A\n"), content="[(2, Mandatory)]")40 debug_inspect(41 line_break_points("\n\n"),42 content="[(1, Mandatory), (2, Mandatory)]",43 )44}4546///|47test "cursor exhaustion and shared iteration" {48 let empty = @segmentation.line_breaks("")49 debug_inspect(empty.opportunity(), content="Prohibited")50 assert_true(empty.next().is_empty())51 assert_true(empty.next().is_empty())52 assert_true(empty.iter().collect().is_empty())53 debug_inspect(empty.opportunity(), content="Prohibited")54 let cursor = @segmentation.line_breaks("A B C")55 debug_inspect(cursor.opportunity(), content="Prohibited")56 inspect(cursor.next(), content="A ")57 debug_inspect(cursor.opportunity(), content="Allowed")58 debug_inspect(cursor.opportunity(), content="Allowed")59 let remaining = []60 for segment in cursor {61 remaining.push((segment.to_owned(), cursor.opportunity()))62 }63 debug_inspect(remaining, content="[(\"B \", Allowed), (\"C\", Mandatory)]")64 assert_true(cursor.next().is_empty())65 assert_true(cursor.next().is_empty())66 assert_true(cursor.iter().collect().is_empty())67 debug_inspect(cursor.opportunity(), content="Mandatory")68}6970///|71test "UTF-16 offsets relative to an independent view" {72 let source = "x😀 中z"73 let text = source.unsafe_view(start=1, end=5)74 let cursor = @segmentation.line_breaks(text)75 let first = cursor.next()76 inspect(first, content="😀 ")77 assert_eq(first.start_offset(), 1)78 assert_eq(first.length(), 3)79 debug_inspect(cursor.opportunity(), content="Allowed")80 let second = cursor.next()81 inspect(second, content="中")82 assert_eq(second.start_offset(), 4)83 debug_inspect(cursor.opportunity(), content="Mandatory")84 let end = cursor.next()85 assert_true(end.is_empty())86 assert_eq(end.start_offset(), 5)87 debug_inspect(88 line_break_points(text),89 content="[(3, Allowed), (4, Mandatory)]",90 )91 // A quote at the start of the view must not inherit the preceding CJK.92 let quoted = "中’中"93 debug_inspect(94 line_break_points(quoted.unsafe_view(start=1, end=quoted.length())),95 content="[(2, Mandatory)]",96 )97 debug_inspect(98 line_break_points("🇺🇳🇨🇳"),99 content="[(4, Allowed), (8, Mandatory)]",100 )101}102103///|104test "default rules do not impose grapheme boundaries or trim whitespace" {105 debug_inspect(106 line_break_points(" \u{0308}a"),107 content="[(1, Allowed), (3, Mandatory)]",108 )109 debug_inspect(110 line_break_points("a b "),111 content="[(3, Allowed), (5, Mandatory)]",112 )113 debug_inspect(line_break_points("a\u{A0}b"), content="[(3, Mandatory)]")114 debug_inspect(115 line_break_points("a\u{200B} b"),116 content="[(4, Allowed), (5, Mandatory)]",117 )118}119120///|121test "unpaired surrogate halves resolve to alphabetic" {122 let text = "a😀b"123 debug_inspect(124 line_break_points(text.unsafe_view(start=0, end=2)),125 content="[(2, Mandatory)]",126 )127 debug_inspect(128 line_break_points(text.unsafe_view(start=2, end=text.length())),129 content="[(2, Mandatory)]",130 )131 // A complete pair is an ideograph-like emoji instead of two alphabetics.132 debug_inspect(133 line_break_points(text),134 content="[(1, Allowed), (3, Allowed), (4, Mandatory)]",135 )136}137138///|139test "long contexts preserve combining, numeric, space and RI rules" {140 let marks = "\u{0308}".repeat(10000)141 @debug.assert_eq(line_break_points("a" + marks + "b"), [142 (10002, @segmentation.Mandatory),143 ])144 let punctuation = ",/".repeat(5000)145 @debug.assert_eq(line_break_points("1" + punctuation + "2$"), [146 (10003, @segmentation.Mandatory),147 ])148 let spaces = " ".repeat(10000)149 @debug.assert_eq(line_break_points("(" + spaces + "a"), [150 (10002, @segmentation.Mandatory),151 ])152 // Marks ignored by LB9 must not reset the significant RI count.153 let flags = "🇦\u{0308}".repeat(1001)154 let points = line_break_points(flags)155 assert_eq(points.length(), 501)156 for i in 0..<500 {157 @debug.assert_eq(points[i], ((i + 1) * 6, @segmentation.Allowed))158 }159 @debug.assert_eq(points[500], (3003, @segmentation.Mandatory))160}161162///|163test "Unicode 18 hyphens and potential emoji" {164 // Unicode 18 changes LB12a: BA no longer permits a break before GL.165 debug_inspect(166 line_break_points("\u{2013}\u{A0}x"),167 content="[(3, Mandatory)]",168 )169 debug_inspect(170 line_break_points("\u{AD}\u{A0}x"),171 content="[(1, Allowed), (3, Mandatory)]",172 )173 debug_inspect(174 line_break_points("\u{1FC00}\u{1F3FB}"),175 content="[(4, Mandatory)]",176 )177}178
///|
fn line_break_points(
text : StringView,
) -> Array[(Int32, @segmentation.BreakOpportunity)] {
let cursor = @segmentation.line_breaks(text)
let points = []
for segment in cursor {
points.push(
(
segment.start_offset() - text.start_offset() + segment.length(),
cursor.opportunity(),
),
)
}
points
}
///|
test "allowed, mandatory, and end of text" {
debug_inspect(
line_break_points("Hello world!"),
content="[(6, Allowed), (12, Mandatory)]",
)
debug_inspect(
line_break_points("中文"),
content="[(1, Allowed), (2, Mandatory)]",
)
debug_inspect(
line_break_points("A\r\nB"),
content="[(3, Mandatory), (4, Mandatory)]",
)
for
newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {
@debug.assert_eq(line_break_points("A" + newline + "B"), [
(2, @segmentation.Mandatory),
(3, @segmentation.Mandatory),
])
}
debug_inspect(line_break_points("A\n"), content="[(2, Mandatory)]")
debug_inspect(
line_break_points("\n\n"),
content="[(1, Mandatory), (2, Mandatory)]",
)
}
///|
test "cursor exhaustion and shared iteration" {
let empty = @segmentation.line_breaks("")
debug_inspect(empty.opportunity(), content="Prohibited")
assert_true(empty.next().is_empty())
assert_true(empty.next().is_empty())
assert_true(empty.iter().collect().is_empty())
debug_inspect(empty.opportunity(), content="Prohibited")
let cursor = @segmentation.line_breaks("A B C")
debug_inspect(cursor.opportunity(), content="Prohibited")
inspect(cursor.next(), content="A ")
debug_inspect(cursor.opportunity(), content="Allowed")
debug_inspect(cursor.opportunity(), content="Allowed")
let remaining = []
for segment in cursor {
remaining.push((segment.to_owned(), cursor.opportunity()))
}
debug_inspect(remaining, content="[(\"B \", Allowed), (\"C\", Mandatory)]")
assert_true(cursor.next().is_empty())
assert_true(cursor.next().is_empty())
assert_true(cursor.iter().collect().is_empty())
debug_inspect(cursor.opportunity(), content="Mandatory")
}
///|
test "UTF-16 offsets relative to an independent view" {
let source = "x😀 中z"
let text = source.unsafe_view(start=1, end=5)
let cursor = @segmentation.line_breaks(text)
let first = cursor.next()
inspect(first, content="😀 ")
assert_eq(first.start_offset(), 1)
assert_eq(first.length(), 3)
debug_inspect(cursor.opportunity(), content="Allowed")
let second = cursor.next()
inspect(second, content="中")
assert_eq(second.start_offset(), 4)
debug_inspect(cursor.opportunity(), content="Mandatory")
let end = cursor.next()
assert_true(end.is_empty())
assert_eq(end.start_offset(), 5)
debug_inspect(
line_break_points(text),
content="[(3, Allowed), (4, Mandatory)]",
)
// A quote at the start of the view must not inherit the preceding CJK.
let quoted = "中’中"
debug_inspect(
line_break_points(quoted.unsafe_view(start=1, end=quoted.length())),
content="[(2, Mandatory)]",
)
debug_inspect(
line_break_points("🇺🇳🇨🇳"),
content="[(4, Allowed), (8, Mandatory)]",
)
}
///|
test "default rules do not impose grapheme boundaries or trim whitespace" {
debug_inspect(
line_break_points(" \u{0308}a"),
content="[(1, Allowed), (3, Mandatory)]",
)
debug_inspect(
line_break_points("a b "),
content="[(3, Allowed), (5, Mandatory)]",
)
debug_inspect(line_break_points("a\u{A0}b"), content="[(3, Mandatory)]")
debug_inspect(
line_break_points("a\u{200B} b"),
content="[(4, Allowed), (5, Mandatory)]",
)
}
///|
test "unpaired surrogate halves resolve to alphabetic" {
let text = "a😀b"
debug_inspect(
line_break_points(text.unsafe_view(start=0, end=2)),
content="[(2, Mandatory)]",
)
debug_inspect(
line_break_points(text.unsafe_view(start=2, end=text.length())),
content="[(2, Mandatory)]",
)
// A complete pair is an ideograph-like emoji instead of two alphabetics.
debug_inspect(
line_break_points(text),
content="[(1, Allowed), (3, Allowed), (4, Mandatory)]",
)
}
///|
test "long contexts preserve combining, numeric, space and RI rules" {
let marks = "\u{0308}".repeat(10000)
@debug.assert_eq(line_break_points("a" + marks + "b"), [
(10002, @segmentation.Mandatory),
])
let punctuation = ",/".repeat(5000)
@debug.assert_eq(line_break_points("1" + punctuation + "2$"), [
(10003, @segmentation.Mandatory),
])
let spaces = " ".repeat(10000)
@debug.assert_eq(line_break_points("(" + spaces + "a"), [
(10002, @segmentation.Mandatory),
])
// Marks ignored by LB9 must not reset the significant RI count.
let flags = "🇦\u{0308}".repeat(1001)
let points = line_break_points(flags)
assert_eq(points.length(), 501)
for i in 0..<500 {
@debug.assert_eq(points[i], ((i + 1) * 6, @segmentation.Allowed))
}
@debug.assert_eq(points[500], (3003, @segmentation.Mandatory))
}
///|
test "Unicode 18 hyphens and potential emoji" {
// Unicode 18 changes LB12a: BA no longer permits a break before GL.
debug_inspect(
line_break_points("\u{2013}\u{A0}x"),
content="[(3, Mandatory)]",
)
debug_inspect(
line_break_points("\u{AD}\u{A0}x"),
content="[(1, Allowed), (3, Mandatory)]",
)
debug_inspect(
line_break_points("\u{1FC00}\u{1F3FB}"),
content="[(4, Mandatory)]",
)
}