unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/grapheme_test.mbt
1///|2/// Describe each cluster by its code points, so that the expected output3/// stays readable for combining marks, controls, and emoji sequences.4fn cluster_code_points(text : StringView) -> Array[Array[String]] {5  @segmentation.graphemes(text)6  .iter()7  .map(cluster => {8    cluster9    .iter()10    .map(character => "U+" + character.to_int().to_string(radix=16).to_upper())11    .collect()12  })13  .collect()14}1516///|17test "empty text has no clusters" {18  debug_inspect(19    cluster_code_points(""),20    content=(21      #|[]22    ),23  )24}2526///|27test "ASCII breaks between every character" {28  debug_inspect(29    cluster_code_points("ab 1"),30    content=(31      #|[["U+61"], ["U+62"], ["U+20"], ["U+31"]]32    ),33  )34}3536///|37test "CR LF is one cluster and controls stand alone" {38  debug_inspect(39    cluster_code_points("a\r\nb\n\r\u{0}\u{301}"),40    content=(41      #|[42      #|  ["U+61"],43      #|  ["U+D", "U+A"],44      #|  ["U+62"],45      #|  ["U+A"],46      #|  ["U+D"],47      #|  ["U+0"],48      #|  ["U+301"],49      #|]50    ),51  )52}5354///|55test "combining marks and spacing marks extend the base" {56  debug_inspect(57    cluster_code_points("e\u{301}\u{323}o\u{308}\u{0915}\u{093E}"),58    content=(59      #|[["U+65", "U+301", "U+323"], ["U+6F", "U+308"], ["U+915", "U+93E"]]60    ),61  )62}6364///|65test "prepend characters attach to the following character" {66  debug_inspect(67    cluster_code_points("\u{0600}1\u{0600}\n"),68    content=(69      #|[["U+600", "U+31"], ["U+600"], ["U+A"]]70    ),71  )72}7374///|75test "Hangul syllable sequences" {76  // L V T jamo, a precomposed LV syllable followed by T, and LVT followed77  // by V, which starts a new cluster.78  debug_inspect(79    cluster_code_points(80      "\u{1100}\u{1161}\u{11A8}\u{AC00}\u{11A8}\u{AC01}\u{1161}",81    ),82    content=(83      #|[84      #|  ["U+1100", "U+1161", "U+11A8"],85      #|  ["U+AC00", "U+11A8"],86      #|  ["U+AC01"],87      #|  ["U+1161"],88      #|]89    ),90  )91}9293///|94test "regional indicators pair up from the start of a run" {95  debug_inspect(96    cluster_code_points("πŸ‡ΊπŸ‡³πŸ‡¨πŸ‡³πŸ‡―aπŸ‡―πŸ‡΅"),97    content=(98      #|[99      #|  ["U+1F1FA", "U+1F1F3"],100      #|  ["U+1F1E8", "U+1F1F3"],101      #|  ["U+1F1EF"],102      #|  ["U+61"],103      #|  ["U+1F1EF", "U+1F1F5"],104      #|]105    ),106  )107}108109///|110test "emoji modifier and ZWJ sequences" {111  debug_inspect(112    cluster_code_points("πŸ‘πŸ½πŸ‘¨β€πŸ‘©β€πŸ‘§β€οΈβ€πŸ”₯"),113    content=(114      #|[115      #|  ["U+1F44D", "U+1F3FD"],116      #|  ["U+1F468", "U+200D", "U+1F469", "U+200D", "U+1F467"],117      #|  ["U+2764", "U+FE0F", "U+200D", "U+1F525"],118      #|]119    ),120  )121}122123///|124test "ZWJ joins only after an extended pictographic" {125  debug_inspect(126    cluster_code_points("a\u{200D}πŸ‘§πŸ‘¨\u{200D}a"),127    content=(128      #|[["U+61", "U+200D"], ["U+1F467"], ["U+1F468", "U+200D"], ["U+61"]]129    ),130  )131}132133///|134test "Indic conjuncts stay together across a linker" {135  // KA VIRAMA SSA is one conjunct; without the virama they are separate.136  debug_inspect(137    cluster_code_points("\u{0915}\u{094D}\u{0937}\u{0915}\u{0937}"),138    content=(139      #|[["U+915", "U+94D", "U+937"], ["U+915"], ["U+937"]]140    ),141  )142  // Extenders around the linker, including ZWJ, keep the conjunct open.143  debug_inspect(144    cluster_code_points("\u{0915}\u{093C}\u{094D}\u{200D}\u{0937}"),145    content=(146      #|[["U+915", "U+93C", "U+94D", "U+200D", "U+937"]]147    ),148  )149}150151///|152test "a linker joins the next consonant without a leading consonant" {153  // Since Unicode 18, GB9c no longer requires a consonant before the linker.154  debug_inspect(155    cluster_code_points("a\u{094D}\u{0924}"),156    content=(157      #|[["U+61", "U+94D", "U+924"]]158    ),159  )160  // VEDIC SIGN JIHVAMULIYA is a linker that is not Extend, so it starts the161  // cluster itself.162  debug_inspect(163    cluster_code_points("a\u{1CF5}\u{0915}\u{1CF5}a"),164    content=(165      #|[["U+61"], ["U+1CF5", "U+915"], ["U+1CF5"], ["U+61"]]166    ),167  )168}169170///|171test "clusters are views into the input" {172  let text = "xe\u{301}πŸ‡ΊπŸ‡³y"173  let offsets = @segmentation.graphemes(text[1:])174    .iter()175    .map(cluster => (cluster.start_offset(), cluster.length()))176    .collect()177  debug_inspect(178    offsets,179    content=(180      #|[(1, 2), (3, 4), (7, 1)]181    ),182  )183}184185///|186test "unpaired surrogates behave like ordinary characters" {187  // Slicing by code unit offsets can split a surrogate pair. Each half is a188  // base character of its own, so the low half takes the combining mark.189  let text = "aπŸ˜€\u{301}"190  debug_inspect(191    @segmentation.graphemes(text.unsafe_view(start=0, end=2))192    .iter()193    .map(cluster => cluster.length())194    .collect(),195    content=(196      #|[1, 1]197    ),198  )199  debug_inspect(200    @segmentation.graphemes(text.unsafe_view(start=2, end=text.length()))201    .iter()202    .map(cluster => cluster.length())203    .collect(),204    content=(205      #|[2]206    ),207  )208}209210///|211test "next returns an empty view at the end" {212  let text = "xe\u{301}y"213  let clusters = @segmentation.graphemes(text[1:])214  let first = clusters.next()215  debug_inspect(216    (first.to_owned(), first.start_offset()),217    content=(218      #|("é", 1)219    ),220  )221  let second = clusters.next()222  debug_inspect(223    (second.to_owned(), second.start_offset()),224    content=(225      #|("y", 3)226    ),227  )228  // The empty view sits at the end of the text, and `next` keeps returning it.229  let end = clusters.next()230  debug_inspect(231    (end.is_empty(), end.start_offset()),232    content=(233      #|(true, 4)234    ),235  )236  debug_inspect(237    clusters.next().is_empty(),238    content=(239      #|true240    ),241  )242  inspect(243    @segmentation.graphemes("").next().is_empty(),244    content=(245      #|true246    ),247  )248}249250///|251test "iter and next share the cursor" {252  let clusters = @segmentation.graphemes("abcd")253  inspect(254    clusters.next(),255    content=(256      #|a257    ),258  )259  let iter = clusters.iter()260  debug_inspect(261    iter.next().map(cluster => cluster.to_owned()),262    content=(263      #|Some("b")264    ),265  )266  inspect(267    clusters.next(),268    content=(269      #|c270    ),271  )272  debug_inspect(273    iter.map(cluster => cluster.to_owned()).collect(),274    content=(275      #|["d"]276    ),277  )278  inspect(279    clusters.next().is_empty(),280    content=(281      #|true282    ),283  )284}285286///|287test "for .. in iterates the clusters" {288  let lengths = []289  for cluster in @segmentation.graphemes("e\u{301}πŸ‡ΊπŸ‡³x") {290    lengths.push(cluster.length())291  }292  debug_inspect(293    lengths,294    content=(295      #|[2, 4, 1]296    ),297  )298}299