unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/line_break_test.mbt
1///|2fn line_break_points(3  text : StringView,4) -> Array[(Int32, @segmentation.BreakOpportunity)] {5  let cursor = @segmentation.line_breaks(text)6  let points = []7  for segment in cursor {8    points.push(9      (10        segment.start_offset() - text.start_offset() + segment.length(),11        cursor.opportunity(),12      ),13    )14  }15  points16}1718///|19test "allowed, mandatory, and end of text" {20  debug_inspect(21    line_break_points("Hello world!"),22    content="[(6, Allowed), (12, Mandatory)]",23  )24  debug_inspect(25    line_break_points("中文"),26    content="[(1, Allowed), (2, Mandatory)]",27  )28  debug_inspect(29    line_break_points("A\r\nB"),30    content="[(3, Mandatory), (4, Mandatory)]",31  )32  for33    newline in ["\r", "\n", "\u{85}", "\u{B}", "\u{C}", "\u{2028}", "\u{2029}"] {34    @debug.assert_eq(line_break_points("A" + newline + "B"), [35      (2, @segmentation.Mandatory),36      (3, @segmentation.Mandatory),37    ])38  }39  debug_inspect(line_break_points("A\n"), content="[(2, Mandatory)]")40  debug_inspect(41    line_break_points("\n\n"),42    content="[(1, Mandatory), (2, Mandatory)]",43  )44}4546///|47test "cursor exhaustion and shared iteration" {48  let empty = @segmentation.line_breaks("")49  debug_inspect(empty.opportunity(), content="Prohibited")50  assert_true(empty.next().is_empty())51  assert_true(empty.next().is_empty())52  assert_true(empty.iter().collect().is_empty())53  debug_inspect(empty.opportunity(), content="Prohibited")54  let cursor = @segmentation.line_breaks("A B C")55  debug_inspect(cursor.opportunity(), content="Prohibited")56  inspect(cursor.next(), content="A ")57  debug_inspect(cursor.opportunity(), content="Allowed")58  debug_inspect(cursor.opportunity(), content="Allowed")59  let remaining = []60  for segment in cursor {61    remaining.push((segment.to_owned(), cursor.opportunity()))62  }63  debug_inspect(remaining, content="[(\"B \", Allowed), (\"C\", Mandatory)]")64  assert_true(cursor.next().is_empty())65  assert_true(cursor.next().is_empty())66  assert_true(cursor.iter().collect().is_empty())67  debug_inspect(cursor.opportunity(), content="Mandatory")68}6970///|71test "UTF-16 offsets relative to an independent view" {72  let source = "x😀 中z"73  let text = source.unsafe_view(start=1, end=5)74  let cursor = @segmentation.line_breaks(text)75  let first = cursor.next()76  inspect(first, content="😀 ")77  assert_eq(first.start_offset(), 1)78  assert_eq(first.length(), 3)79  debug_inspect(cursor.opportunity(), content="Allowed")80  let second = cursor.next()81  inspect(second, content="中")82  assert_eq(second.start_offset(), 4)83  debug_inspect(cursor.opportunity(), content="Mandatory")84  let end = cursor.next()85  assert_true(end.is_empty())86  assert_eq(end.start_offset(), 5)87  debug_inspect(88    line_break_points(text),89    content="[(3, Allowed), (4, Mandatory)]",90  )91  // A quote at the start of the view must not inherit the preceding CJK.92  let quoted = "中’中"93  debug_inspect(94    line_break_points(quoted.unsafe_view(start=1, end=quoted.length())),95    content="[(2, Mandatory)]",96  )97  debug_inspect(98    line_break_points("🇺🇳🇨🇳"),99    content="[(4, Allowed), (8, Mandatory)]",100  )101}102103///|104test "default rules do not impose grapheme boundaries or trim whitespace" {105  debug_inspect(106    line_break_points(" \u{0308}a"),107    content="[(1, Allowed), (3, Mandatory)]",108  )109  debug_inspect(110    line_break_points("a  b "),111    content="[(3, Allowed), (5, Mandatory)]",112  )113  debug_inspect(line_break_points("a\u{A0}b"), content="[(3, Mandatory)]")114  debug_inspect(115    line_break_points("a\u{200B}  b"),116    content="[(4, Allowed), (5, Mandatory)]",117  )118}119120///|121test "unpaired surrogate halves resolve to alphabetic" {122  let text = "a😀b"123  debug_inspect(124    line_break_points(text.unsafe_view(start=0, end=2)),125    content="[(2, Mandatory)]",126  )127  debug_inspect(128    line_break_points(text.unsafe_view(start=2, end=text.length())),129    content="[(2, Mandatory)]",130  )131  // A complete pair is an ideograph-like emoji instead of two alphabetics.132  debug_inspect(133    line_break_points(text),134    content="[(1, Allowed), (3, Allowed), (4, Mandatory)]",135  )136}137138///|139test "long contexts preserve combining, numeric, space and RI rules" {140  let marks = "\u{0308}".repeat(10000)141  @debug.assert_eq(line_break_points("a" + marks + "b"), [142    (10002, @segmentation.Mandatory),143  ])144  let punctuation = ",/".repeat(5000)145  @debug.assert_eq(line_break_points("1" + punctuation + "2$"), [146    (10003, @segmentation.Mandatory),147  ])148  let spaces = " ".repeat(10000)149  @debug.assert_eq(line_break_points("(" + spaces + "a"), [150    (10002, @segmentation.Mandatory),151  ])152  // Marks ignored by LB9 must not reset the significant RI count.153  let flags = "🇦\u{0308}".repeat(1001)154  let points = line_break_points(flags)155  assert_eq(points.length(), 501)156  for i in 0..<500 {157    @debug.assert_eq(points[i], ((i + 1) * 6, @segmentation.Allowed))158  }159  @debug.assert_eq(points[500], (3003, @segmentation.Mandatory))160}161162///|163test "Unicode 18 hyphens and potential emoji" {164  // Unicode 18 changes LB12a: BA no longer permits a break before GL.165  debug_inspect(166    line_break_points("\u{2013}\u{A0}x"),167    content="[(3, Mandatory)]",168  )169  debug_inspect(170    line_break_points("\u{AD}\u{A0}x"),171    content="[(1, Allowed), (3, Mandatory)]",172  )173  debug_inspect(174    line_break_points("\u{1FC00}\u{1F3FB}"),175    content="[(4, Mandatory)]",176  )177}178