segmentation/word_rules.mbt1///|2fn word_ignored(property : Int32) -> Bool {3 property is (WbExtend | WbFormat | WbZWJ)4}56///|7fn word_letter(property : Int32) -> Bool {8 property is (WbALetter | WbHebrewLetter)9}1011///|12fn word_newline(property : Int32) -> Bool {13 property is (WbCR | WbLF | WbNewline)14}1516///|17fn word_segment_ends(text : StringView) -> Array[Int32] {18 let input = text_break_input(text, word_break_value)19 let properties = input.properties20 let length = properties.length()21 // Nearest non-ignored characters on each side of a position (WB4).22 // Preparing these once avoids rescanning arbitrarily long Extend runs.23 let before = Array::make(length + 1, -1)24 let after = Array::make(length + 1, length)25 for i in 0..<length {26 before[i + 1] = if word_ignored(properties[i]) { before[i] } else { i }27 }28 for i in length>..0 {29 after[i] = if word_ignored(properties[i]) { after[i + 1] } else { i }30 }31 let ends = []32 let mut regional_count = 033 for i in 1..<length {34 let left = properties[i - 1]35 if !word_ignored(left) {36 regional_count = if left == WbRegionalIndicator {37 regional_count + 138 } else {39 040 }41 }42 if word_boundary(input, before, after, i, regional_count) {43 ends.push(input.ends[i - 1])44 }45 }46 if length > 0 {47 ends.push(text.length())48 }49 ends50}5152///|53/// Apply WB3-WB999 in order at an interior code point boundary.54fn word_boundary(55 input : TextBreakInput,56 before : Array[Int32],57 after : Array[Int32],58 index : Int32,59 regional_count : Int32,60) -> Bool {61 let properties = input.properties62 let raw_left = properties[index - 1]63 let right = properties[index]64 // WB3, WB3a, WB3b: these precede the ignore rule.65 if raw_left == WbCR && right == WbLF {66 return false67 }68 if word_newline(raw_left) || word_newline(right) {69 return true70 }71 // WB3c and WB3d use adjacent characters, not WB4's reduced context.72 if raw_left == WbZWJ &&73 (grapheme_break_value(input.points[index]) & ExtendedPictographicBit) != 0 {74 return false75 }76 if raw_left == WbWSegSpace && right == WbWSegSpace {77 return false78 }79 // WB4: leading ignored characters form their own segment.80 if word_ignored(right) {81 return false82 }83 let left_index = before[index]84 if left_index < 0 {85 return true86 }87 let left = properties[left_index]88 let previous_index = before[left_index]89 let previous = if previous_index < 0 { 0 } else { properties[previous_index] }90 let following_index = after[index + 1]91 let following = if following_index == properties.length() {92 093 } else {94 properties[following_index]95 }96 // WB5-WB7: letters and medial punctuation.97 if word_letter(left) && word_letter(right) {98 return false99 }100 if word_letter(left) &&101 right is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&102 word_letter(following) {103 return false104 }105 if word_letter(previous) &&106 left is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&107 word_letter(right) {108 return false109 }110 // WB7a-WB7c: Hebrew quotes.111 if left == WbHebrewLetter && right == WbSingleQuote {112 return false113 }114 if left == WbHebrewLetter &&115 right == WbDoubleQuote &&116 following == WbHebrewLetter {117 return false118 }119 if previous == WbHebrewLetter &&120 left == WbDoubleQuote &&121 right == WbHebrewLetter {122 return false123 }124 // WB8-WB12: numbers, adjoining letters, and numeric punctuation.125 if left == WbNumeric && right == WbNumeric {126 return false127 }128 if (word_letter(left) && right == WbNumeric) ||129 (left == WbNumeric && word_letter(right)) {130 return false131 }132 if previous == WbNumeric &&133 left is (WbMidNum | WbMidNumLet | WbSingleQuote) &&134 right == WbNumeric {135 return false136 }137 if left == WbNumeric &&138 right is (WbMidNum | WbMidNumLet | WbSingleQuote) &&139 following == WbNumeric {140 return false141 }142 // WB13-WB13b: Katakana and connector punctuation.143 if left == WbKatakana && right == WbKatakana {144 return false145 }146 if left147 is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana | WbExtendNumLet) &&148 right == WbExtendNumLet {149 return false150 }151 if left == WbExtendNumLet &&152 right is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana) {153 return false154 }155 // WB15-WB16: pair regional indicators, ignoring WB4 characters.156 if left == WbRegionalIndicator &&157 right == WbRegionalIndicator &&158 regional_count % 2 == 1 {159 return false160 }161 true // WB999162}163
///|
fn word_ignored(property : Int32) -> Bool {
property is (WbExtend | WbFormat | WbZWJ)
}
///|
fn word_letter(property : Int32) -> Bool {
property is (WbALetter | WbHebrewLetter)
}
///|
fn word_newline(property : Int32) -> Bool {
property is (WbCR | WbLF | WbNewline)
}
///|
fn word_segment_ends(text : StringView) -> Array[Int32] {
let input = text_break_input(text, word_break_value)
let properties = input.properties
let length = properties.length()
// Nearest non-ignored characters on each side of a position (WB4).
// Preparing these once avoids rescanning arbitrarily long Extend runs.
let before = Array::make(length + 1, -1)
let after = Array::make(length + 1, length)
for i in 0..<length {
before[i + 1] = if word_ignored(properties[i]) { before[i] } else { i }
}
for i in length>..0 {
after[i] = if word_ignored(properties[i]) { after[i + 1] } else { i }
}
let ends = []
let mut regional_count = 0
for i in 1..<length {
let left = properties[i - 1]
if !word_ignored(left) {
regional_count = if left == WbRegionalIndicator {
regional_count + 1
} else {
0
}
}
if word_boundary(input, before, after, i, regional_count) {
ends.push(input.ends[i - 1])
}
}
if length > 0 {
ends.push(text.length())
}
ends
}
///|
/// Apply WB3-WB999 in order at an interior code point boundary.
fn word_boundary(
input : TextBreakInput,
before : Array[Int32],
after : Array[Int32],
index : Int32,
regional_count : Int32,
) -> Bool {
let properties = input.properties
let raw_left = properties[index - 1]
let right = properties[index]
// WB3, WB3a, WB3b: these precede the ignore rule.
if raw_left == WbCR && right == WbLF {
return false
}
if word_newline(raw_left) || word_newline(right) {
return true
}
// WB3c and WB3d use adjacent characters, not WB4's reduced context.
if raw_left == WbZWJ &&
(grapheme_break_value(input.points[index]) & ExtendedPictographicBit) != 0 {
return false
}
if raw_left == WbWSegSpace && right == WbWSegSpace {
return false
}
// WB4: leading ignored characters form their own segment.
if word_ignored(right) {
return false
}
let left_index = before[index]
if left_index < 0 {
return true
}
let left = properties[left_index]
let previous_index = before[left_index]
let previous = if previous_index < 0 { 0 } else { properties[previous_index] }
let following_index = after[index + 1]
let following = if following_index == properties.length() {
0
} else {
properties[following_index]
}
// WB5-WB7: letters and medial punctuation.
if word_letter(left) && word_letter(right) {
return false
}
if word_letter(left) &&
right is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&
word_letter(following) {
return false
}
if word_letter(previous) &&
left is (WbMidLetter | WbMidNumLet | WbSingleQuote) &&
word_letter(right) {
return false
}
// WB7a-WB7c: Hebrew quotes.
if left == WbHebrewLetter && right == WbSingleQuote {
return false
}
if left == WbHebrewLetter &&
right == WbDoubleQuote &&
following == WbHebrewLetter {
return false
}
if previous == WbHebrewLetter &&
left == WbDoubleQuote &&
right == WbHebrewLetter {
return false
}
// WB8-WB12: numbers, adjoining letters, and numeric punctuation.
if left == WbNumeric && right == WbNumeric {
return false
}
if (word_letter(left) && right == WbNumeric) ||
(left == WbNumeric && word_letter(right)) {
return false
}
if previous == WbNumeric &&
left is (WbMidNum | WbMidNumLet | WbSingleQuote) &&
right == WbNumeric {
return false
}
if left == WbNumeric &&
right is (WbMidNum | WbMidNumLet | WbSingleQuote) &&
following == WbNumeric {
return false
}
// WB13-WB13b: Katakana and connector punctuation.
if left == WbKatakana && right == WbKatakana {
return false
}
if left
is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana | WbExtendNumLet) &&
right == WbExtendNumLet {
return false
}
if left == WbExtendNumLet &&
right is (WbALetter | WbHebrewLetter | WbNumeric | WbKatakana) {
return false
}
// WB15-WB16: pair regional indicators, ignoring WB4 characters.
if left == WbRegionalIndicator &&
right == WbRegionalIndicator &&
regional_count % 2 == 1 {
return false
}
true // WB999
}