unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/grapheme_line_break_test.mbt
1///|2fn grapheme_breaks(3  text : StringView,4) -> Array[(String, @segmentation.BreakOpportunity)] {5  let cursor = @segmentation.graphemes(text)6  let results = []7  for cluster in cursor {8    results.push((cluster.to_owned(), cursor.line_break()))9  }10  results11}1213///|14test "grapheme line break kinds" {15  debug_inspect(16    grapheme_breaks("ab cd\n"),17    content=(18      #|[19      #|  ("a", Prohibited),20      #|  ("b", Prohibited),21      #|  (" ", Allowed),22      #|  ("c", Prohibited),23      #|  ("d", Prohibited),24      #|  ("\n", Mandatory),25      #|]26    ),27  )28  debug_inspect(29    grapheme_breaks("中文"),30    content="[(\"中\", Allowed), (\"文\", Mandatory)]",31  )32  debug_inspect(33    grapheme_breaks("A\r\nB"),34    content=(35      #|[("A", Prohibited), ("\r\n", Mandatory), ("B", Mandatory)]36    ),37  )38  for39    newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {40    @debug.assert_eq(grapheme_breaks("A" + newline + "B"), [41      ("A", @segmentation.Prohibited),42      (newline, @segmentation.Mandatory),43      ("B", @segmentation.Mandatory),44    ])45  }46}4748///|49test "grapheme line break query does not advance and survives exhaustion" {50  let empty = @segmentation.graphemes("")51  debug_inspect(empty.line_break(), content="Prohibited")52  assert_true(empty.next().is_empty())53  debug_inspect(empty.line_break(), content="Prohibited")54  let cursor = @segmentation.graphemes("a b")55  debug_inspect(cursor.line_break(), content="Prohibited")56  inspect(cursor.next(), content="a")57  debug_inspect(cursor.line_break(), content="Prohibited")58  debug_inspect(cursor.line_break(), content="Prohibited")59  let iterator = cursor.iter()60  debug_inspect(61    iterator.next(),62    content=(63      #|Some(<StringView: " ">)64    ),65  )66  debug_inspect(cursor.line_break(), content="Allowed")67  debug_inspect(cursor.line_break(), content="Allowed")68  inspect(cursor.next(), content="b")69  debug_inspect(cursor.line_break(), content="Mandatory")70  assert_true(iterator.next() is None)71  assert_true(cursor.next().is_empty())72  assert_true(cursor.next().is_empty())73  debug_inspect(cursor.line_break(), content="Mandatory")74}7576///|77test "line break queries can start late or skip clusters" {78  let cursor = @segmentation.graphemes("a b c")79  inspect(cursor.next(), content="a")80  inspect(cursor.next(), content=" ")81  debug_inspect(cursor.line_break(), content="Allowed")82  inspect(cursor.next(), content="b")83  inspect(cursor.next(), content=" ")84  debug_inspect(cursor.line_break(), content="Allowed")85  inspect(cursor.next(), content="c")86  debug_inspect(cursor.line_break(), content="Mandatory")87  let late = @segmentation.graphemes("x")88  assert_eq(late.iter().count(), 1)89  assert_true(late.next().is_empty())90  debug_inspect(late.line_break(), content="Mandatory")91}9293///|94test "grapheme line breaking preserves whole clusters and default rule context" {95  // UAX #14 allows a break after SPACE inside the first UAX #29 cluster.96  // That opportunity is suppressed, not moved to the cluster's end.97  debug_inspect(98    grapheme_breaks(" \u{0308}a"),99    content="[(\" ̈\", Prohibited), (\"a\", Mandatory)]",100  )101  debug_inspect(102    grapheme_breaks("e\u{0301}x"),103    content="[(\"é\", Prohibited), (\"x\", Mandatory)]",104  )105  debug_inspect(106    grapheme_breaks("🇺🇳🇨🇳"),107    content="[(\"🇺🇳\", Allowed), (\"🇨🇳\", Mandatory)]",108  )109  debug_inspect(110    grapheme_breaks("👨‍👩‍👧x"),111    content="[(\"👨‍👩‍👧\", Allowed), (\"x\", Mandatory)]",112  )113}114115///|116test "grapheme line breaking uses independent UTF-16 views" {117  let source = "x😀 中z"118  let text = source.unsafe_view(start=1, end=5)119  debug_inspect(120    grapheme_breaks(text),121    content="[(\"😀\", Prohibited), (\" \", Allowed), (\"中\", Mandatory)]",122  )123  let quoted = "中’中"124  debug_inspect(125    grapheme_breaks(quoted.unsafe_view(start=1, end=quoted.length())),126    content="[(\"’\", Prohibited), (\"中\", Mandatory)]",127  )128  let pair = "😀"129  for130    half in [131      pair.unsafe_view(start=0, end=1),132      pair.unsafe_view(start=1, end=pair.length()),133    ] {134    let cursor = @segmentation.graphemes(half)135    let cluster = cursor.next()136    assert_eq(cluster.start_offset(), half.start_offset())137    assert_eq(cluster.length(), 1)138    debug_inspect(cursor.line_break(), content="Mandatory")139    assert_true(cursor.next().is_empty())140  }141}142