unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/line_break.mbt
1///|2/// Line_Break values in the order of the `LineBreak` enum, with the3/// description UAX #14 gives each class. LineBreak.txt writes the short4/// aliases, which are also the constructor names.5let line_break_values : ReadOnlyArray[(String, String)] = [6  ("AI", "Ambiguous (Alphabetic or Ideographic)"),7  ("AK", "Aksara"),8  ("AL", "Alphabetic"),9  ("AP", "Aksara Pre-Base"),10  ("AS", "Aksara Start"),11  ("B2", "Break Opportunity Before and After"),12  ("BA", "Break After"),13  ("BB", "Break Before"),14  ("BK", "Mandatory Break"),15  ("CB", "Contingent Break Opportunity"),16  ("CJ", "Conditional Japanese Starter"),17  ("CL", "Close Punctuation"),18  ("CM", "Combining Mark"),19  ("CP", "Close Parenthesis"),20  ("CR", "Carriage Return"),21  ("EB", "Emoji Base"),22  ("EM", "Emoji Modifier"),23  ("EX", "Exclamation/Interrogation"),24  ("GL", "Non-breaking (\"Glue\")"),25  ("H2", "Hangul LV Syllable"),26  ("H3", "Hangul LVT Syllable"),27  ("HH", "Unambiguous Hyphen"),28  ("HL", "Hebrew Letter"),29  ("HY", "Hyphen"),30  ("ID", "Ideographic"),31  ("IN", "Inseparable"),32  ("IS", "Infix Numeric Separator"),33  ("JL", "Hangul L Jamo"),34  ("JT", "Hangul T Jamo"),35  ("JV", "Hangul V Jamo"),36  ("LF", "Line Feed"),37  ("NL", "Next Line"),38  ("NS", "Nonstarter"),39  ("NU", "Numeric"),40  ("OP", "Open Punctuation"),41  ("PO", "Postfix Numeric"),42  ("PR", "Prefix Numeric"),43  ("QU", "Quotation"),44  ("RI", "Regional Indicator"),45  ("SA", "Complex Context Dependent (South East Asian)"),46  ("SG", "Surrogate"),47  ("SP", "Space"),48  ("SY", "Symbols Allowing Break After"),49  ("VF", "Virama Final"),50  ("VI", "Virama"),51  ("WJ", "Word Joiner"),52  ("XX", "Unknown"),53  ("ZW", "Zero Width Space"),54  ("ZWJ", "Zero Width Joiner"),55]5657///|58/// Number of bits holding a Line_Break ordinal in the packed high ranges.59const LineBreakBits : Int32 = 66061///|62/// Emit Line_Break data from `LineBreak.txt` as a three-level trie for planes63/// 0-3, with packed ranges for the remaining planes.64///65/// Every value in `line_break_values` must occur in the file, so that a66/// value Unicode withdraws does not linger in the enum.67pub fn emit_line_break(content : String) -> String raise {68  let values = @parsing.parse_enumerated_ordinals(69    content,70    property="Line_Break",71    default="XX",72    name => line_break_values.search_by(entry => name == entry.0),73  )74  let used = FixedArray::make(line_break_values.length(), false)75  for ordinal in values {76    used[ordinal] = true77  }78  for ordinal, entry in line_break_values {79    guard used[ordinal] else {80      fail("Line_Break value \{entry.0} does not occur in LineBreak.txt")81    }82  }83  let out = @emitter.Emitter::new()84  out.write_file_header("Line_Break lookup data", "LineBreak.txt")85  write_enumerated_trie(86    out,87    property="Line_Break",88    prefix="line_break",89    enum_name="LineBreak",90    value_count=line_break_values.length(),91    value_bits=LineBreakBits,92    values,93  )94  out.finish()95}9697///|98/// Emit the `LineBreak` enum and the `line_break` lookup of the root package,99/// which reads the ordinals `emit_line_break` stores.100pub fn emit_line_break_enum() -> String raise {101  let out = @emitter.Emitter::new()102  out.write_file_header("Line_Break values and lookup", "LineBreak.txt")103  out.write_enum(104    type_name="LineBreak",105    doc=[106      "Unicode Line_Break values", "See UAX #14: https://www.unicode.org/reports/tr14/#Table1",107    ],108    converter="ordinal_to_line_break",109    values=line_break_values110      .iter()111      .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })112      .collect(),113  )114  out.write(115    (116      #|///|117      #|/// Get the Line_Break for a character.118      #|///119      #|/// Line_Break is the property the line breaking rules of UAX #14 are120      #|/// written in terms of. It is a property of a single character, and the121      #|/// value is the one the Unicode Character Database assigns: rule LB1 of122      #|/// the algorithm, which resolves AI, CJ, SA, SG, and XX to other classes,123      #|/// is not applied. Unassigned code points are XX, except in ranges124      #|/// reserved for characters of a known class, such as ID in the CJK125      #|/// ideograph blocks and in planes 2 and 3.126      #|pub fn line_break(c : Char) -> LineBreak {127      #|  ordinal_to_line_break(@line_break.lookup_line_break(c))128      #|}129      #|130    ),131  )132  out.finish()133}134