1///|2/// Shared storage for the word and sentence cursors. Only segment ends are3/// retained after boundary analysis; slices keep the original backing text.4priv struct TextSegments {5 text : StringView6 ends : Array[Int32]7 mut index : Int328 mut offset : Int329}1011///|12fn TextSegments::next(self : TextSegments) -> StringView {13 let start = self.offset14 if self.index < self.ends.length() {15 self.offset = self.ends[self.index]16 self.index += 117 }18 // Every entry of `ends` is a boundary recorded by the analysis, so the slice19 // is in range and never splits a character: no clamping is needed.20 self.text.unsafe_view(start~, end=self.offset)21}2223///|24fn TextSegments::iter(self : TextSegments) -> Iter[StringView] {25 Iter::new(() => {26 let segment = self.next()27 if segment.is_empty() {28 None29 } else {30 Some(segment)31 }32 })33}3435///|36/// One entry per decoded code point, with UTF-16 end offsets. Unpaired37/// surrogates have the unlisted (Other) property; valid pairs stay together.38priv struct TextBreakInput {39 points : Array[Int32]40 properties : Array[Int32]41 ends : Array[Int32]42}4344///|45fn text_break_input(46 text : StringView,47 property : (Char) -> Int32,48) -> TextBreakInput {49 let points = []50 let properties = []51 let ends = []52 let mut offset = 053 while offset < text.length() {54 let cp = code_point_at(text, offset, text.length())55 offset += if cp >= 0x10000 { 2 } else { 1 }56 points.push(cp)57 properties.push(58 if cp >= 0xD800 && cp <= 0xDFFF {59 060 } else {61 property(cp.unsafe_to_char())62 },63 )64 ends.push(offset)65 }66 { points, properties, ends, }67}68
///|
/// Shared storage for the word and sentence cursors. Only segment ends are
/// retained after boundary analysis; slices keep the original backing text.
priv struct TextSegments {
text : StringView
ends : Array[Int32]
mut index : Int32
mut offset : Int32
}
///|
fn TextSegments::next(self : TextSegments) -> StringView {
let start = self.offset
if self.index < self.ends.length() {
self.offset = self.ends[self.index]
self.index += 1
}
// Every entry of `ends` is a boundary recorded by the analysis, so the slice
// is in range and never splits a character: no clamping is needed.
self.text.unsafe_view(start~, end=self.offset)
}
///|
fn TextSegments::iter(self : TextSegments) -> Iter[StringView] {
Iter::new(() => {
let segment = self.next()
if segment.is_empty() {
None
} else {
Some(segment)
}
})
}
///|
/// One entry per decoded code point, with UTF-16 end offsets. Unpaired
/// surrogates have the unlisted (Other) property; valid pairs stay together.
priv struct TextBreakInput {
points : Array[Int32]
properties : Array[Int32]
ends : Array[Int32]
}
///|
fn text_break_input(
text : StringView,
property : (Char) -> Int32,
) -> TextBreakInput {
let points = []
let properties = []
let ends = []
let mut offset = 0
while offset < text.length() {
let cp = code_point_at(text, offset, text.length())
offset += if cp >= 0x10000 { 2 } else { 1 }
points.push(cp)
properties.push(
if cp >= 0xD800 && cp <= 0xDFFF {
0
} else {
property(cp.unsafe_to_char())
},
)
ends.push(offset)
}
{ points, properties, ends, }
}