unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/word_test.mbt
1///|2test "word segments retain punctuation, spaces, and numeric context" {3  @debug.assert_eq(4    @segmentation.word_segments(5      "can't  3.14 foo_bar אב\"ג カタカナ\r\n中文",6    )7    .iter()8    .map(s => s.to_owned())9    .collect(),10    [11      "can't", "  ", "3.14", " ", "foo_bar", " ", "אב\"ג", " ", "カタカナ",12      "\r\n", "中", "文",13    ],14  )15}1617///|18test "words filter by Alphabetic or Number and preserve complete segments" {19  @debug.assert_eq(20    @segmentation.words(21      "👩‍👩‍👧‍👦 -- can't 42 Ⅷ ² _ \u{301} e\u{301} 中文",22    )23    .iter()24    .map(s => s.to_owned())25    .collect(),26    ["can't", "42", "Ⅷ", "²", "e\u{301}", "中", "文"],27  )28  assert_true(@segmentation.words("!? 👩‍👧 🇺🇳").next().is_empty())29  @debug.assert_eq(30    @segmentation.words("1️⃣ #️⃣")31    .iter()32    .map(s => s.to_owned())33    .collect(),34    ["1️⃣"],35  )36}3738///|39test "word cursor exhaustion and interleaved iterators" {40  let cursor = @segmentation.word_segments("one two!")41  let iterator = cursor.iter()42  inspect(cursor.next(), content="one")43  @debug.assert_eq(iterator.next(), Some(" "))44  inspect(cursor.next(), content="two")45  @debug.assert_eq(iterator.next(), Some("!"))46  assert_true(cursor.next().is_empty())47  assert_true(cursor.next().is_empty())48  @debug.assert_eq(iterator.next(), None)49  let filtered = @segmentation.words("one two!")50  let iterator = filtered.iter()51  inspect(filtered.next(), content="one")52  @debug.assert_eq(iterator.next(), Some("two"))53  let end = filtered.next()54  assert_true(end.is_empty())55  assert_eq(end.start_offset(), 8)56  assert_true(filtered.next().is_empty())57  @debug.assert_eq(iterator.next(), None)58  let empty_segments = @segmentation.word_segments("")59  let empty_words = @segmentation.words("")60  assert_true(empty_segments.next().is_empty())61  assert_true(empty_segments.next().is_empty())62  assert_true(empty_words.next().is_empty())63  assert_true(empty_words.next().is_empty())64}6566///|67test "word views retain UTF-16 offsets and use independent context" {68  let backing = "xa.b😀z"69  let text = backing.unsafe_view(start=2, end=6)70  let cursor = @segmentation.word_segments(text)71  @debug.assert_eq(72    cursor73    .iter()74    .map(s => (s.to_owned(), s.start_offset(), s.length()))75    .collect(),76    [(".", 2, 1), ("b", 3, 1), ("😀", 4, 2)],77  )78  assert_eq(cursor.next().start_offset(), 6)79  let flags = "🇦🇧🇨🇩🇪"80  @debug.assert_eq(81    @segmentation.word_segments(flags.unsafe_view(start=2, end=flags.length()))82    .iter()83    .map(s => s.to_owned())84    .collect(),85    ["🇧🇨", "🇩🇪"],86  )87}8889///|90test "word ignored characters and RI pairing keep the required context" {91  @debug.assert_eq(92    @segmentation.word_segments("\u{301}a\r\n\u{301}b a\u{301}'\u{301}b")93    .iter()94    .map(s => s.to_owned())95    .collect(),96    ["\u{301}", "a", "\r\n", "\u{301}", "b", " ", "a\u{301}'\u{301}b"],97  )98  @debug.assert_eq(99    @segmentation.word_segments("🇦\u{301}🇧🇨\u{301}🇩🇪")100    .iter()101    .map(s => s.to_owned())102    .collect(),103    ["🇦\u{301}🇧", "🇨\u{301}🇩", "🇪"],104  )105  // WB3d is before WB4: marks between spaces prevent their grouping.106  @debug.assert_eq(107    @segmentation.word_segments(" \u{301} ")108    .iter()109    .map(s => s.to_owned())110    .collect(),111    [" \u{301}", " "],112  )113}114115///|116test "word boundaries can split an extended grapheme" {117  let text = "\u{6DD} "118  assert_eq(@segmentation.graphemes(text).iter().count(), 1)119  @debug.assert_eq(120    @segmentation.word_segments(text).iter().map(s => s.to_owned()).collect(),121    ["\u{6DD}", " "],122  )123}124125///|126test "word cursors preserve unpaired surrogates" {127  let text = "a😀\u{301}b"128  let leading = text.unsafe_view(start=0, end=2)129  @debug.assert_eq(130    @segmentation.word_segments(leading).iter().map(s => s.length()).collect(),131    [1, 1],132  )133  let trailing = text.unsafe_view(start=2, end=text.length())134  @debug.assert_eq(135    @segmentation.word_segments(trailing).iter().map(s => s.length()).collect(),136    [2, 1],137  )138  @debug.assert_eq(139    @segmentation.words(trailing).iter().map(s => s.to_owned()).collect(),140    ["b"],141  )142}143144///|145test "long ignored runs do not change word punctuation lookaround" {146  let marks = "\u{301}".repeat(20000)147  let text = "a" + marks + "'" + marks + "b"148  let cursor = @segmentation.word_segments(text)149  assert_eq(cursor.next().length(), text.length())150  assert_true(cursor.next().is_empty())151}152