unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/word_rules.mbt
1///|2fn word_ignored(property : Int32) -> Bool {3  property is (WbExtend | WbFormat | WbZWJ)4}56///|7fn word_letter(property : Int32) -> Bool {8  property is (WbALetter | WbHebrewLetter)9}1011///|12fn word_newline(property : Int32) -> Bool {13  property is (WbCR | WbLF | WbNewline)14}1516///|17fn word_segment_ends(text : StringView) -> Array[Int32] {18  let input = text_break_input(text, word_break_value)19  let properties = input.properties20  let length = properties.length()21  // Nearest non-ignored characters on each side of a position (WB4).22  // Preparing these once avoids rescanning arbitrarily long Extend runs.23  let before = Array::make(length + 1, -1)24  let after = Array::make(length + 1, length)25  for i in 0..<length {26    before[i + 1] = if word_ignored(properties[i]) { before[i] } else { i }27  }28  for i in length>..0 {29    after[i] = if word_ignored(properties[i]) { after[i + 1] } else { i }30  }31  let ends = []32  let mut regional_count = 033  for i in 1..<length {34    let left = properties[i - 1]35    if !word_ignored(left) {36      regional_count = if left == WbRegionalIndicator {37        regional_count + 138      } else {39        040      }41    }42    if word_boundary(input, before, after, i, regional_count) {43      ends.push(input.ends[i - 1])44    }45  }46  if length > 0 {47    ends.push(text.length())48  }49  ends50}5152///|53/// Apply WB3-WB999 in order at an interior code point boundary.54fn word_boundary(55  input : TextBreakInput,56  before : Array[Int32],57  after : Array[Int32],58  index : Int32,59  regional_count : Int32,60) -> Bool {61  let properties = input.properties62  let raw_left = properties[index - 1]63  let right = properties[index]64  // WB3, WB3a, WB3b: these precede the ignore rule.65  if raw_left == WbCR && right == WbLF {66    return false67  }68  if word_newline(raw_left) || word_newline(right) {69    return true70  }71  // WB3c and WB3d use adjacent characters, not WB4's reduced context.72  if raw_left == WbZWJ &&73    (grapheme_break_value(input.points[index]) & ExtendedPictographicBit) != 0 {74    return false75  }76  if raw_left == WbWSegSpace && right == WbWSegSpace {77    return false78  }79  // WB4: leading ignored characters form their own segment.80  if word_ignored(right) {81    return false82  }83  let left_index = before[index]84  if left_index < 0 {85    return true86  }87  let left = properties[left_index]88  let previous_index = before[left_index]89  let previous = if previous_index < 0 { 0 } else { properties[previous_index] }90  let following_index = after[index + 1]91  let following = if following_index == properties.length() {92    093  } else {94    properties[following_index]95  }96  // WB5-WB7: letters and medial punctuation.97  if word_letter(left) && word_letter(right) {98    return false99  }100  if word_letter(left) &&101    right is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&102    word_letter(following) {103    return false104  }105  if word_letter(previous) &&106    left is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&107    word_letter(right) {108    return false109  }110  // WB7a-WB7c: Hebrew quotes.111  if left == WbHebrewLetter && right == WbSingleQuote {112    return false113  }114  if left == WbHebrewLetter &&115    right == WbDoubleQuote &&116    following == WbHebrewLetter {117    return false118  }119  if previous == WbHebrewLetter &&120    left == WbDoubleQuote &&121    right == WbHebrewLetter {122    return false123  }124  // WB8-WB12: numbers, adjoining letters, and numeric punctuation.125  if left == WbNumeric && right == WbNumeric {126    return false127  }128  if (word_letter(left) && right == WbNumeric) ||129    (left == WbNumeric && word_letter(right)) {130    return false131  }132  if previous == WbNumeric &&133    left is (WbMidNum | WbMidNumLet | WbSingleQuote) &&134    right == WbNumeric {135    return false136  }137  if left == WbNumeric &&138    right is (WbMidNum | WbMidNumLet | WbSingleQuote) &&139    following == WbNumeric {140    return false141  }142  // WB13-WB13b: Katakana and connector punctuation.143  if left == WbKatakana && right == WbKatakana {144    return false145  }146  if left147    is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana | WbExtendNumLet) &&148    right == WbExtendNumLet {149    return false150  }151  if left == WbExtendNumLet &&152    right is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana) {153    return false154  }155  // WB15-WB16: pair regional indicators, ignoring WB4 characters.156  if left == WbRegionalIndicator &&157    right == WbRegionalIndicator &&158    regional_count % 2 == 1 {159    return false160  }161  true // WB999162}163