tools/ucd/line_break.mbt1///|2/// Line_Break values in the order of the `LineBreak` enum, with the3/// description UAX #14 gives each class. LineBreak.txt writes the short4/// aliases, which are also the constructor names.5let line_break_values : ReadOnlyArray[(String, String)] = [6 ("AI", "Ambiguous (Alphabetic or Ideographic)"),7 ("AK", "Aksara"),8 ("AL", "Alphabetic"),9 ("AP", "Aksara Pre-Base"),10 ("AS", "Aksara Start"),11 ("B2", "Break Opportunity Before and After"),12 ("BA", "Break After"),13 ("BB", "Break Before"),14 ("BK", "Mandatory Break"),15 ("CB", "Contingent Break Opportunity"),16 ("CJ", "Conditional Japanese Starter"),17 ("CL", "Close Punctuation"),18 ("CM", "Combining Mark"),19 ("CP", "Close Parenthesis"),20 ("CR", "Carriage Return"),21 ("EB", "Emoji Base"),22 ("EM", "Emoji Modifier"),23 ("EX", "Exclamation/Interrogation"),24 ("GL", "Non-breaking (\"Glue\")"),25 ("H2", "Hangul LV Syllable"),26 ("H3", "Hangul LVT Syllable"),27 ("HH", "Unambiguous Hyphen"),28 ("HL", "Hebrew Letter"),29 ("HY", "Hyphen"),30 ("ID", "Ideographic"),31 ("IN", "Inseparable"),32 ("IS", "Infix Numeric Separator"),33 ("JL", "Hangul L Jamo"),34 ("JT", "Hangul T Jamo"),35 ("JV", "Hangul V Jamo"),36 ("LF", "Line Feed"),37 ("NL", "Next Line"),38 ("NS", "Nonstarter"),39 ("NU", "Numeric"),40 ("OP", "Open Punctuation"),41 ("PO", "Postfix Numeric"),42 ("PR", "Prefix Numeric"),43 ("QU", "Quotation"),44 ("RI", "Regional Indicator"),45 ("SA", "Complex Context Dependent (South East Asian)"),46 ("SG", "Surrogate"),47 ("SP", "Space"),48 ("SY", "Symbols Allowing Break After"),49 ("VF", "Virama Final"),50 ("VI", "Virama"),51 ("WJ", "Word Joiner"),52 ("XX", "Unknown"),53 ("ZW", "Zero Width Space"),54 ("ZWJ", "Zero Width Joiner"),55]5657///|58/// Number of bits holding a Line_Break ordinal in the packed high ranges.59const LineBreakBits : Int32 = 66061///|62/// Emit Line_Break data from `LineBreak.txt` as a three-level trie for planes63/// 0-3, with packed ranges for the remaining planes.64///65/// Every value in `line_break_values` must occur in the file, so that a66/// value Unicode withdraws does not linger in the enum.67pub fn emit_line_break(content : String) -> String raise {68 let values = @parsing.parse_enumerated_ordinals(69 content,70 property="Line_Break",71 default="XX",72 name => line_break_values.search_by(entry => name == entry.0),73 )74 let used = FixedArray::make(line_break_values.length(), false)75 for ordinal in values {76 used[ordinal] = true77 }78 for ordinal, entry in line_break_values {79 guard used[ordinal] else {80 fail("Line_Break value \{entry.0} does not occur in LineBreak.txt")81 }82 }83 let out = @emitter.Emitter::new()84 out.write_file_header("Line_Break lookup data", "LineBreak.txt")85 write_enumerated_trie(86 out,87 property="Line_Break",88 prefix="line_break",89 enum_name="LineBreak",90 value_count=line_break_values.length(),91 value_bits=LineBreakBits,92 values,93 )94 out.finish()95}9697///|98/// Emit the `LineBreak` enum and the `line_break` lookup of the root package,99/// which reads the ordinals `emit_line_break` stores.100pub fn emit_line_break_enum() -> String raise {101 let out = @emitter.Emitter::new()102 out.write_file_header("Line_Break values and lookup", "LineBreak.txt")103 out.write_enum(104 type_name="LineBreak",105 doc=[106 "Unicode Line_Break values", "See UAX #14: https://www.unicode.org/reports/tr14/#Table1",107 ],108 converter="ordinal_to_line_break",109 values=line_break_values110 .iter()111 .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })112 .collect(),113 )114 out.write(115 (116 #|///|117 #|/// Get the Line_Break for a character.118 #|///119 #|/// Line_Break is the property the line breaking rules of UAX #14 are120 #|/// written in terms of. It is a property of a single character, and the121 #|/// value is the one the Unicode Character Database assigns: rule LB1 of122 #|/// the algorithm, which resolves AI, CJ, SA, SG, and XX to other classes,123 #|/// is not applied. Unassigned code points are XX, except in ranges124 #|/// reserved for characters of a known class, such as ID in the CJK125 #|/// ideograph blocks and in planes 2 and 3.126 #|pub fn line_break(c : Char) -> LineBreak {127 #| ordinal_to_line_break(@line_break.lookup_line_break(c))128 #|}129 #|130 ),131 )132 out.finish()133}134
///|
/// Line_Break values in the order of the `LineBreak` enum, with the
/// description UAX #14 gives each class. LineBreak.txt writes the short
/// aliases, which are also the constructor names.
let line_break_values : ReadOnlyArray[(String, String)] = [
("AI", "Ambiguous (Alphabetic or Ideographic)"),
("AK", "Aksara"),
("AL", "Alphabetic"),
("AP", "Aksara Pre-Base"),
("AS", "Aksara Start"),
("B2", "Break Opportunity Before and After"),
("BA", "Break After"),
("BB", "Break Before"),
("BK", "Mandatory Break"),
("CB", "Contingent Break Opportunity"),
("CJ", "Conditional Japanese Starter"),
("CL", "Close Punctuation"),
("CM", "Combining Mark"),
("CP", "Close Parenthesis"),
("CR", "Carriage Return"),
("EB", "Emoji Base"),
("EM", "Emoji Modifier"),
("EX", "Exclamation/Interrogation"),
("GL", "Non-breaking (\"Glue\")"),
("H2", "Hangul LV Syllable"),
("H3", "Hangul LVT Syllable"),
("HH", "Unambiguous Hyphen"),
("HL", "Hebrew Letter"),
("HY", "Hyphen"),
("ID", "Ideographic"),
("IN", "Inseparable"),
("IS", "Infix Numeric Separator"),
("JL", "Hangul L Jamo"),
("JT", "Hangul T Jamo"),
("JV", "Hangul V Jamo"),
("LF", "Line Feed"),
("NL", "Next Line"),
("NS", "Nonstarter"),
("NU", "Numeric"),
("OP", "Open Punctuation"),
("PO", "Postfix Numeric"),
("PR", "Prefix Numeric"),
("QU", "Quotation"),
("RI", "Regional Indicator"),
("SA", "Complex Context Dependent (South East Asian)"),
("SG", "Surrogate"),
("SP", "Space"),
("SY", "Symbols Allowing Break After"),
("VF", "Virama Final"),
("VI", "Virama"),
("WJ", "Word Joiner"),
("XX", "Unknown"),
("ZW", "Zero Width Space"),
("ZWJ", "Zero Width Joiner"),
]
///|
/// Number of bits holding a Line_Break ordinal in the packed high ranges.
const LineBreakBits : Int32 = 6
///|
/// Emit Line_Break data from `LineBreak.txt` as a three-level trie for planes
/// 0-3, with packed ranges for the remaining planes.
///
/// Every value in `line_break_values` must occur in the file, so that a
/// value Unicode withdraws does not linger in the enum.
pub fn emit_line_break(content : String) -> String raise {
let values = @parsing.parse_enumerated_ordinals(
content,
property="Line_Break",
default="XX",
name => line_break_values.search_by(entry => name == entry.0),
)
let used = FixedArray::make(line_break_values.length(), false)
for ordinal in values {
used[ordinal] = true
}
for ordinal, entry in line_break_values {
guard used[ordinal] else {
fail("Line_Break value \{entry.0} does not occur in LineBreak.txt")
}
}
let out = @emitter.Emitter::new()
out.write_file_header("Line_Break lookup data", "LineBreak.txt")
write_enumerated_trie(
out,
property="Line_Break",
prefix="line_break",
enum_name="LineBreak",
value_count=line_break_values.length(),
value_bits=LineBreakBits,
values,
)
out.finish()
}
///|
/// Emit the `LineBreak` enum and the `line_break` lookup of the root package,
/// which reads the ordinals `emit_line_break` stores.
pub fn emit_line_break_enum() -> String raise {
let out = @emitter.Emitter::new()
out.write_file_header("Line_Break values and lookup", "LineBreak.txt")
out.write_enum(
type_name="LineBreak",
doc=[
"Unicode Line_Break values", "See UAX #14: https://www.unicode.org/reports/tr14/#Table1",
],
converter="ordinal_to_line_break",
values=line_break_values
.iter()
.map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.1, })
.collect(),
)
out.write(
(
#|///|
#|/// Get the Line_Break for a character.
#|///
#|/// Line_Break is the property the line breaking rules of UAX #14 are
#|/// written in terms of. It is a property of a single character, and the
#|/// value is the one the Unicode Character Database assigns: rule LB1 of
#|/// the algorithm, which resolves AI, CJ, SA, SG, and XX to other classes,
#|/// is not applied. Unassigned code points are XX, except in ranges
#|/// reserved for characters of a known class, such as ID in the CJK
#|/// ideograph blocks and in planes 2 and 3.
#|pub fn line_break(c : Char) -> LineBreak {
#| ordinal_to_line_break(@line_break.lookup_line_break(c))
#|}
#|
),
)
out.finish()
}