unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/segmentation_data/text_break_data.mbt
1///|2let word_break_names : ReadOnlyArray[String] = [3  "Other", "CR", "LF", "Newline", "Extend", "ZWJ", "Regional_Indicator", "Format",4  "Katakana", "Hebrew_Letter", "ALetter", "Single_Quote", "Double_Quote", "MidNumLet",5  "MidLetter", "MidNum", "Numeric", "ExtendNumLet", "WSegSpace",6]78///|9let sentence_break_names : ReadOnlyArray[String] = [10  "Other", "CR", "LF", "Sep", "Extend", "Format", "Sp", "Lower", "Upper", "OLetter",11  "Numeric", "ATerm", "STerm", "Close", "SContinue",12]1314///|15fn emit_text_break(16  source : String,17  property~ : String,18  file~ : String,19  prefix~ : String,20  constant_prefix~ : String,21  names~ : ReadOnlyArray[String],22  bits~ : Int32,23) -> String raise {24  let values = @parsing.parse_enumerated_ordinals(25    source,26    property~,27    default="Other",28    name => names.search_by(n => name == n),29  )30  let out = @emitter.Emitter::new()31  out.write_file_header(32    "\{property} properties for Unicode Text Segmentation (UAX #29)",33    file,34  )35  for code, name in names {36    if code != 0 {37      let constant_name = name.replace_all(old="_", new="")38      out.write_line("///|")39      out.write_line("/// \{property}=\{name}")40      out.write_line(41        "const \{constant_prefix}\{constant_name} : Int = \{code}\n",42      )43    }44  }45  out.write_value_trie(46    property~,47    prefix~,48    lookup="fn \{prefix}_value",49    lookup_doc=[50      "Look up \{property}; unlisted code points have value zero (Other).",51    ],52    value_bits=bits,53    values,54  )55  out.finish()56}5758///|59/// Generate the default word boundary property table from WordBreakProperty.txt.60pub fn emit_word_break(source : String) -> String raise {61  emit_text_break(62    source,63    property="Word_Break",64    file="WordBreakProperty.txt",65    prefix="word_break",66    constant_prefix="Wb",67    names=word_break_names,68    bits=5,69  )70}7172///|73/// Generate the sentence boundary property table from SentenceBreakProperty.txt.74pub fn emit_sentence_break(source : String) -> String raise {75  emit_text_break(76    source,77    property="Sentence_Break",78    file="SentenceBreakProperty.txt",79    prefix="sentence_break",80    constant_prefix="Sb",81    names=sentence_break_names,82    bits=4,83  )84}85