tools/segmentation_data/text_break_data.mbt1///|2let word_break_names : ReadOnlyArray[String] = [3 "Other", "CR", "LF", "Newline", "Extend", "ZWJ", "Regional_Indicator", "Format",4 "Katakana", "Hebrew_Letter", "ALetter", "Single_Quote", "Double_Quote", "MidNumLet",5 "MidLetter", "MidNum", "Numeric", "ExtendNumLet", "WSegSpace",6]78///|9let sentence_break_names : ReadOnlyArray[String] = [10 "Other", "CR", "LF", "Sep", "Extend", "Format", "Sp", "Lower", "Upper", "OLetter",11 "Numeric", "ATerm", "STerm", "Close", "SContinue",12]1314///|15fn emit_text_break(16 source : String,17 property~ : String,18 file~ : String,19 prefix~ : String,20 constant_prefix~ : String,21 names~ : ReadOnlyArray[String],22 bits~ : Int32,23) -> String raise {24 let values = @parsing.parse_enumerated_ordinals(25 source,26 property~,27 default="Other",28 name => names.search_by(n => name == n),29 )30 let out = @emitter.Emitter::new()31 out.write_file_header(32 "\{property} properties for Unicode Text Segmentation (UAX #29)",33 file,34 )35 for code, name in names {36 if code != 0 {37 let constant_name = name.replace_all(old="_", new="")38 out.write_line("///|")39 out.write_line("/// \{property}=\{name}")40 out.write_line(41 "const \{constant_prefix}\{constant_name} : Int = \{code}\n",42 )43 }44 }45 out.write_value_trie(46 property~,47 prefix~,48 lookup="fn \{prefix}_value",49 lookup_doc=[50 "Look up \{property}; unlisted code points have value zero (Other).",51 ],52 value_bits=bits,53 values,54 )55 out.finish()56}5758///|59/// Generate the default word boundary property table from WordBreakProperty.txt.60pub fn emit_word_break(source : String) -> String raise {61 emit_text_break(62 source,63 property="Word_Break",64 file="WordBreakProperty.txt",65 prefix="word_break",66 constant_prefix="Wb",67 names=word_break_names,68 bits=5,69 )70}7172///|73/// Generate the sentence boundary property table from SentenceBreakProperty.txt.74pub fn emit_sentence_break(source : String) -> String raise {75 emit_text_break(76 source,77 property="Sentence_Break",78 file="SentenceBreakProperty.txt",79 prefix="sentence_break",80 constant_prefix="Sb",81 names=sentence_break_names,82 bits=4,83 )84}85
///|
let word_break_names : ReadOnlyArray[String] = [
"Other", "CR", "LF", "Newline", "Extend", "ZWJ", "Regional_Indicator", "Format",
"Katakana", "Hebrew_Letter", "ALetter", "Single_Quote", "Double_Quote", "MidNumLet",
"MidLetter", "MidNum", "Numeric", "ExtendNumLet", "WSegSpace",
]
///|
let sentence_break_names : ReadOnlyArray[String] = [
"Other", "CR", "LF", "Sep", "Extend", "Format", "Sp", "Lower", "Upper", "OLetter",
"Numeric", "ATerm", "STerm", "Close", "SContinue",
]
///|
fn emit_text_break(
source : String,
property~ : String,
file~ : String,
prefix~ : String,
constant_prefix~ : String,
names~ : ReadOnlyArray[String],
bits~ : Int32,
) -> String raise {
let values = @parsing.parse_enumerated_ordinals(
source,
property~,
default="Other",
name => names.search_by(n => name == n),
)
let out = @emitter.Emitter::new()
out.write_file_header(
"\{property} properties for Unicode Text Segmentation (UAX #29)",
file,
)
for code, name in names {
if code != 0 {
let constant_name = name.replace_all(old="_", new="")
out.write_line("///|")
out.write_line("/// \{property}=\{name}")
out.write_line(
"const \{constant_prefix}\{constant_name} : Int = \{code}\n",
)
}
}
out.write_value_trie(
property~,
prefix~,
lookup="fn \{prefix}_value",
lookup_doc=[
"Look up \{property}; unlisted code points have value zero (Other).",
],
value_bits=bits,
values,
)
out.finish()
}
///|
/// Generate the default word boundary property table from WordBreakProperty.txt.
pub fn emit_word_break(source : String) -> String raise {
emit_text_break(
source,
property="Word_Break",
file="WordBreakProperty.txt",
prefix="word_break",
constant_prefix="Wb",
names=word_break_names,
bits=5,
)
}
///|
/// Generate the sentence boundary property table from SentenceBreakProperty.txt.
pub fn emit_sentence_break(source : String) -> String raise {
emit_text_break(
source,
property="Sentence_Break",
file="SentenceBreakProperty.txt",
prefix="sentence_break",
constant_prefix="Sb",
names=sentence_break_names,
bits=4,
)
}