unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/linebreak_tests/linebreak_test.mbt
1///|2/// Default UAX #14 conformance, Unicode 18.0.0:3/// https://www.unicode.org/Public/18.0.0/ucd/auxiliary/LineBreakTest.txt4/// Every non-comment line is executed; no failing or surrogate cases are5/// silently skipped. Offsets in the fixture are converted to UTF-16.6async test "LineBreakTest.txt conformance" {7  let content = @support.read_data_file("LineBreakTest.txt")8  assert_true(content.has_prefix("# LineBreakTest-18.0.0.txt"))9  let mut cases = 010  for line_number, raw_line in content.split("\n") {11    let line = raw_line.split_once("#").map(p => p.0).unwrap_or(raw_line).trim()12    if line.is_empty() {13      continue14    }15    let out = StringBuilder()16    let expected : Array[Int32] = []17    let mut offset = 018    let mut expect_marker = true19    for token in line.split(" ") {20      let token = token.trim()21      if token.is_empty() {22        continue23      }24      if expect_marker {25        match token {26          "÷" => expected.push(offset)27          "×" => ()28          _ => fail("Expected boundary marker on line \{line_number + 1}")29        }30      } else {31        let cp = @support.parse_hex_int(token)32        guard cp >= 0 && cp <= 0x10FFFF && !(cp >= 0xD800 && cp <= 0xDFFF) else {33          fail(34            "Unsupported test code point on line \{line_number + 1}: \{token}",35          )36        }37        out.write_char(cp.unsafe_to_char())38        offset += if cp >= 0x10000 { 2 } else { 1 }39      }40      expect_marker = !expect_marker41    }42    assert_false(expect_marker, msg="Missing final boundary marker")43    let text = out.to_string().to_string_view()44    let breaks = @segmentation.line_breaks(text)45    let actual = []46    let kinds = []47    let mut previous_end = text.start_offset()48    for segment in breaks {49      assert_eq(segment.start_offset(), previous_end)50      assert_false(segment.is_empty())51      previous_end += segment.length()52      actual.push(previous_end - text.start_offset())53      kinds.push(breaks.opportunity())54    }55    assert_eq(previous_end, text.start_offset() + text.length())56    @debug.assert_eq(57      actual,58      expected,59      msg="LineBreakTest.txt line \{line_number + 1}: \{line}",60    )61    // The combined cursor must expose exactly the default opportunities at62    // complete UAX #29 cluster ends, without changing either segmentation.63    let graphemes = @segmentation.graphemes(text)64    let mut point = 065    for grapheme in graphemes {66      let end = grapheme.start_offset() -67        text.start_offset() +68        grapheme.length()69      while point < expected.length() && expected[point] < end {70        point += 171      }72      let kind = if point < expected.length() && expected[point] == end {73        kinds[point]74      } else {75        @segmentation.Prohibited76      }77      @debug.assert_eq(78        graphemes.line_break(),79        kind,80        msg="Grapheme ending at \{end}, LineBreakTest.txt line \{line_number + 1}",81      )82    }83    cases += 184  }85  // Exact fixture count: deleting/skipping any test must fail.86  assert_eq(cases, 19346)87}88