unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

segmentation/sentence_rules.mbt
1///|2fn sentence_paragraph(property : Int32) -> Bool {3  property is (SbSep | SbCR | SbLF)4}56///|7fn sentence_segment_ends(text : StringView) -> Array[Int32] {8  let input = text_break_input(text, sentence_break_value)9  let properties = input.properties10  let length = properties.length()11  // SB8 looks past any number of characters outside its exclusion set.12  // Cache whether the first character in that set is Lower, for O(n) work13  // even on long runs of punctuation and ignored characters.14  let lower_ahead = Array::make(length + 1, false)15  for i in length>..0 {16    lower_ahead[i] = match properties[i] {17      SbLower => true18      SbOLetter | SbUpper | SbSep | SbCR | SbLF | SbATerm | SbSTerm => false19      _ => lower_ahead[i + 1]20    }21  }22  let ends = []23  let mut previous = 024  let mut previous_previous = 025  // Terminator at the start of a suffix matching SATerm Close* or26  // SATerm Close* Sp*, respectively. Zero means there is no such suffix.27  let mut close_term = 028  let mut space_term = 029  for i in 1..<length {30    let left = properties[i - 1]31    let right = properties[i]32    if !(left is (SbExtend | SbFormat)) {33      previous_previous = previous34      previous = left35      close_term = match left {36        SbATerm | SbSTerm => left37        SbClose => close_term38        _ => 039      }40      if left != SbSp {41        space_term = close_term42      }43    }44    let boundary = if left == SbCR && right == SbLF {45      false // SB346    } else if sentence_paragraph(left) {47      true // SB448    } else if right is (SbExtend | SbFormat) {49      false // SB550    } else if previous == SbATerm && right == SbNumeric {51      false // SB652    } else if previous_previous is (SbUpper | SbLower) &&53      previous == SbATerm &&54      right == SbUpper {55      false // SB756    } else if space_term == SbATerm && lower_ahead[i] {57      false // SB858    } else if space_term != 0 && right is (SbSContinue | SbATerm | SbSTerm) {59      false // SB8a60    } else if close_term != 0 &&61      (right is (SbClose | SbSp) || sentence_paragraph(right)) {62      false // SB963    } else if space_term != 0 && (right == SbSp || sentence_paragraph(right)) {64      false // SB1065    } else {66      space_term != 0 // SB11, otherwise SB99867    }68    if boundary {69      ends.push(input.ends[i - 1])70    }71  }72  if length > 0 {73    ends.push(text.length())74  }75  ends76}77