tests/linebreak_tests/linebreak_test.mbt1///|2/// Default UAX #14 conformance, Unicode 18.0.0:3/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/LineBreakTest.txt4/// Every non-comment line is executed; no failing or surrogate cases are5/// silently skipped. Offsets in the fixture are converted to UTF-16.6async test "LineBreakTest.txt conformance" {7 let content = @support.read_data_file("LineBreakTest.txt")8 assert_true(content.has_prefix("# LineBreakTest-18.0.0.txt"))9 let mut cases = 010 for line_number, raw_line in content.split("\n") {11 let line = raw_line.split_once("#").map(p => p.0).unwrap_or(raw_line).trim()12 if line.is_empty() {13 continue14 }15 let out = StringBuilder()16 let expected : Array[Int32] = []17 let mut offset = 018 let mut expect_marker = true19 for token in line.split(" ") {20 let token = token.trim()21 if token.is_empty() {22 continue23 }24 if expect_marker {25 match token {26 "÷" => expected.push(offset)27 "×" => ()28 _ => fail("Expected boundary marker on line \{line_number + 1}")29 }30 } else {31 let cp = @support.parse_hex_int(token)32 guard cp >= 0 && cp <= 0x10FFFF && !(cp >= 0xD800 && cp <= 0xDFFF) else {33 fail(34 "Unsupported test code point on line \{line_number + 1}: \{token}",35 )36 }37 out.write_char(cp.unsafe_to_char())38 offset += if cp >= 0x10000 { 2 } else { 1 }39 }40 expect_marker = !expect_marker41 }42 assert_false(expect_marker, msg="Missing final boundary marker")43 let text = out.to_string().to_string_view()44 let breaks = @segmentation.line_breaks(text)45 let actual = []46 let kinds = []47 let mut previous_end = text.start_offset()48 for segment in breaks {49 assert_eq(segment.start_offset(), previous_end)50 assert_false(segment.is_empty())51 previous_end += segment.length()52 actual.push(previous_end - text.start_offset())53 kinds.push(breaks.opportunity())54 }55 assert_eq(previous_end, text.start_offset() + text.length())56 @debug.assert_eq(57 actual,58 expected,59 msg="LineBreakTest.txt line \{line_number + 1}: \{line}",60 )61 // The combined cursor must expose exactly the default opportunities at62 // complete UAX #29 cluster ends, without changing either segmentation.63 let graphemes = @segmentation.graphemes(text)64 let mut point = 065 for grapheme in graphemes {66 let end = grapheme.start_offset() -67 text.start_offset() +68 grapheme.length()69 while point < expected.length() && expected[point] < end {70 point += 171 }72 let kind = if point < expected.length() && expected[point] == end {73 kinds[point]74 } else {75 @segmentation.Prohibited76 }77 @debug.assert_eq(78 graphemes.line_break(),79 kind,80 msg="Grapheme ending at \{end}, LineBreakTest.txt line \{line_number + 1}",81 )82 }83 cases += 184 }85 // Exact fixture count: deleting/skipping any test must fail.86 assert_eq(cases, 19346)87}88
///|
/// Default UAX #14 conformance, Unicode 18.0.0:
/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/LineBreakTest.txt
/// Every non-comment line is executed; no failing or surrogate cases are
/// silently skipped. Offsets in the fixture are converted to UTF-16.
async test "LineBreakTest.txt conformance" {
let content = @support.read_data_file("LineBreakTest.txt")
assert_true(content.has_prefix("# LineBreakTest-18.0.0.txt"))
let mut cases = 0
for line_number, raw_line in content.split("\n") {
let line = raw_line.split_once("#").map(p => p.0).unwrap_or(raw_line).trim()
if line.is_empty() {
continue
}
let out = StringBuilder()
let expected : Array[Int32] = []
let mut offset = 0
let mut expect_marker = true
for token in line.split(" ") {
let token = token.trim()
if token.is_empty() {
continue
}
if expect_marker {
match token {
"÷" => expected.push(offset)
"×" => ()
_ => fail("Expected boundary marker on line \{line_number + 1}")
}
} else {
let cp = @support.parse_hex_int(token)
guard cp >= 0 && cp <= 0x10FFFF && !(cp >= 0xD800 && cp <= 0xDFFF) else {
fail(
"Unsupported test code point on line \{line_number + 1}: \{token}",
)
}
out.write_char(cp.unsafe_to_char())
offset += if cp >= 0x10000 { 2 } else { 1 }
}
expect_marker = !expect_marker
}
assert_false(expect_marker, msg="Missing final boundary marker")
let text = out.to_string().to_string_view()
let breaks = @segmentation.line_breaks(text)
let actual = []
let kinds = []
let mut previous_end = text.start_offset()
for segment in breaks {
assert_eq(segment.start_offset(), previous_end)
assert_false(segment.is_empty())
previous_end += segment.length()
actual.push(previous_end - text.start_offset())
kinds.push(breaks.opportunity())
}
assert_eq(previous_end, text.start_offset() + text.length())
@debug.assert_eq(
actual,
expected,
msg="LineBreakTest.txt line \{line_number + 1}: \{line}",
)
// The combined cursor must expose exactly the default opportunities at
// complete UAX #29 cluster ends, without changing either segmentation.
let graphemes = @segmentation.graphemes(text)
let mut point = 0
for grapheme in graphemes {
let end = grapheme.start_offset() -
text.start_offset() +
grapheme.length()
while point < expected.length() && expected[point] < end {
point += 1
}
let kind = if point < expected.length() && expected[point] == end {
kinds[point]
} else {
@segmentation.Prohibited
}
@debug.assert_eq(
graphemes.line_break(),
kind,
msg="Grapheme ending at \{end}, LineBreakTest.txt line \{line_number + 1}",
)
}
cases += 1
}
// Exact fixture count: deleting/skipping any test must fail.
assert_eq(cases, 19346)
}