tools/gen/main.mbt1///|2const UnicodeVersion : String = "18.0.0"34///|5/// One generator run: reads Unicode data files, each at most once, and writes6/// generated sources under the repository root.7priv struct Generator {8 root : String9 /// Contents of the data files read so far, by path under10 /// `https://www.unicode.org/Public/<version>/`.11 sources : Map[String, String]12 /// Generated files, relative to `root`.13 written : Array[String]14}1516///|17async fn Generator::new() -> Generator {18 let root = if @fs.exists("moon.work") && @fs.exists("tools/moon.mod") {19 "."20 } else if @fs.exists("../moon.work") && @fs.exists("moon.mod") {21 ".."22 } else {23 fail("Run the generator from the repository root or tools directory")24 }25 { root, sources: Map([]), written: [], }26}2728///|29/// The content of a data file, such as `ucd/UnicodeData.txt`, from the30/// download cache `tools/.cache/<version>/` or else from unicode.org.31async fn Generator::source(self : Generator, path : String) -> String {32 if self.sources.get(path) is Some(content) {33 return content34 }35 let name = match path.rev_split_once("/") {36 Some((_, name)) => name.to_owned()37 None => path38 }39 let cache_directory = "\{self.root}/tools/.cache/\{UnicodeVersion}"40 let cache_path = "\{cache_directory}/\{name}"41 let content = if @fs.exists(cache_path) {42 println("Using cached \{name}")43 @fs.read_text(cache_path)44 } else {45 println("Downloading \{name}...")46 let url = "https://www.unicode.org/Public/\{UnicodeVersion}/\{path}"47 let (response, body) = @http.get(url)48 guard response.code >= 200 && response.code < 300 else {49 fail("Download failed for \{url}: HTTP \{response.code}")50 }51 let content = body.text()52 @fs.mkdir(53 cache_directory,54 permission=0o755,55 recursive=true,56 allow_existing=true,57 )58 @fs.write_text(cache_path, content, permission=0o644)59 content60 }61 self.sources[path] = content62 content63}6465///|66/// The content of a file of the Unicode Character Database.67async fn Generator::ucd(self : Generator, path : String) -> String {68 self.source("ucd/\{path}")69}7071///|72async fn Generator::write(73 self : Generator,74 path : String,75 source : String,76) -> Unit {77 let full_path = "\{self.root}/\{path}"78 if full_path.rev_split_once("/") is Some((directory, _)) {79 @fs.mkdir(80 directory.to_owned(),81 permission=0o755,82 recursive=true,83 allow_existing=true,84 )85 }86 @fs.write_text(full_path, source, permission=0o644)87 self.written.push(path)88}8990///|91async fn Generator::run_moon(92 self : Generator,93 arguments : Array[String],94) -> Unit {95 let status = @shell.Cmd("moon", arguments, cwd=self.root).status()96 guard status == 0 else { fail("moon \{arguments.join(" ")} failed") }97}9899///|100/// Format the generated files and refresh the interfaces of the packages101/// holding them.102async fn Generator::finish(self : Generator) -> Unit {103 self.run_moon(["fmt", ..self.written])104 let packages : Array[String] = []105 for path in self.written {106 if path.rev_split_once("/") is Some((package_, _)) &&107 !packages.contains(package_.to_owned()) {108 packages.push(package_.to_owned())109 }110 }111 self.run_moon(["info", ..packages])112}113114///|115async fn generate_ucd(g : Generator) -> Unit {116 let unicode_data = @ucd.parse_unicode_data(g.ucd("UnicodeData.txt"))117 let exclusions = @ucd.parse_composition_exclusions(118 g.ucd("CompositionExclusions.txt"),119 )120 let normalization_properties = g.ucd("DerivedNormalizationProps.txt")121 g.write(122 "normalization/normalization_data.mbt",123 @ucd.emit_normalization_scan(unicode_data, normalization_properties),124 )125 g.write("ucd/ccc/ccc.mbt", @ucd.emit_ccc())126 g.write(127 "ucd/internal/normalization/normalization.mbt",128 @ucd.emit_normalization(unicode_data, exclusions, normalization_properties),129 )130 g.write(131 "ucd/decomposition/decomposition.mbt",132 @ucd.emit_decomposition(unicode_data),133 )134 g.write(135 "ucd/composition/composition.mbt",136 @ucd.emit_composition(unicode_data, exclusions),137 )138 g.write("ucd/general_category/mark.mbt", @ucd.emit_marks(unicode_data))139 g.write(140 "ucd/general_category/general_category.mbt",141 @ucd.emit_general_category(unicode_data),142 )143 g.write(144 "ucd/punctuation/punctuation.mbt",145 @ucd.emit_punctuation(unicode_data),146 )147 g.write(148 "ucd/case_mapping/case_mapping.mbt",149 @ucd.emit_case_mapping(150 unicode_data,151 @ucd.parse_special_casing(g.ucd("SpecialCasing.txt")),152 ),153 )154 g.write(155 "ucd/case_mapping/case_context_data.mbt",156 @ucd.emit_case_context(g.ucd("DerivedCoreProperties.txt")),157 )158 g.write("ucd/xid/xid.mbt", @ucd.emit_xid(g.ucd("DerivedCoreProperties.txt")))159 // Each binary property lives in a package named after it.160 for161 entry in [162 ("Alphabetic", "DerivedCoreProperties.txt"),163 ("Lowercase", "DerivedCoreProperties.txt"),164 ("Uppercase", "DerivedCoreProperties.txt"),165 ("Default_Ignorable_Code_Point", "DerivedCoreProperties.txt"),166 ("Grapheme_Extend", "DerivedCoreProperties.txt"),167 ("White_Space", "PropList.txt"),168 ("Regional_Indicator", "PropList.txt"),169 ("Prepended_Concatenation_Mark", "PropList.txt"),170 ] {171 let (property, source) = entry172 let package_ = property.to_lower()173 g.write(174 "ucd/\{package_}/\{package_}.mbt",175 @ucd.emit_binary_property(g.ucd(source), property~, source~),176 )177 }178 g.write(179 "ucd/emoji/emoji.mbt",180 @ucd.emit_emoji(181 g.ucd("emoji/emoji-data.txt"),182 g.ucd("emoji/emoji-variation-sequences.txt"),183 ),184 )185 g.write(186 "ucd/east_asian_width/east_asian_width.mbt",187 @ucd.emit_east_asian_width(g.ucd("EastAsianWidth.txt")),188 )189 g.write(190 "ucd/hangul_syllable_type/hangul_syllable_type.mbt",191 @ucd.emit_hangul_syllable_type(g.ucd("HangulSyllableType.txt")),192 )193 g.write(194 "ucd/joining_type/joining_type.mbt",195 @ucd.emit_joining_type(g.ucd("extracted/DerivedJoiningType.txt")),196 )197 let joining_groups = g.ucd("extracted/DerivedJoiningGroup.txt")198 g.write(199 "ucd/joining_group/joining_group.mbt",200 @ucd.emit_joining_group(joining_groups),201 )202 g.write("ucd/joining_group.mbt", @ucd.emit_joining_group_enum(joining_groups))203 g.write(204 "ucd/line_break/line_break.mbt",205 @ucd.emit_line_break(g.ucd("LineBreak.txt")),206 )207 g.write("ucd/line_break.mbt", @ucd.emit_line_break_enum())208}209210///|211async fn generate_casefold(g : Generator) -> Unit {212 g.write(213 "casefold/case_folding_data.mbt",214 @ucd.emit_case_folding(g.ucd("CaseFolding.txt")),215 )216 g.write(217 "casefold/nfkc_casefold_data.mbt",218 @ucd.emit_nfkc_casefold(g.ucd("DerivedNormalizationProps.txt")),219 )220}221222///|223async fn generate_idna(g : Generator) -> Unit {224 g.write(225 "idna/internal/idna/mapping.mbt",226 @idna_data.emit_mapping(g.source("idna/IdnaMappingTable.txt")),227 )228 g.write(229 "idna/internal/idna/idna2008.mbt",230 @idna_data.emit_idna2008(231 g.source("idna/Idna2008.txt"),232 g.ucd("Scripts.txt"),233 ),234 )235}236237///|238async fn generate_bidi(g : Generator) -> Unit {239 g.write(240 "bidi/bidi_class.mbt",241 @bidi_data.emit_bidi_class(g.ucd("extracted/DerivedBidiClass.txt")),242 )243 g.write(244 "bidi/internal/bidi/mirrored.mbt",245 @bidi_data.emit_mirrored(g.ucd("BidiMirroring.txt")),246 )247 g.write(248 "bidi/internal/bidi/bracket.mbt",249 @bidi_data.emit_bracket(g.ucd("BidiBrackets.txt")),250 )251}252253///|254async fn generate_segmentation(g : Generator) -> Unit {255 g.write(256 "segmentation/grapheme_break_property.mbt",257 @segmentation_data.emit_grapheme_break(258 g.ucd("auxiliary/GraphemeBreakProperty.txt"),259 g.ucd("emoji/emoji-data.txt"),260 g.ucd("DerivedCoreProperties.txt"),261 ),262 )263 g.write(264 "segmentation/word_break_property.mbt",265 @segmentation_data.emit_word_break(g.ucd("auxiliary/WordBreakProperty.txt")),266 )267 g.write(268 "segmentation/sentence_break_property.mbt",269 @segmentation_data.emit_sentence_break(270 g.ucd("auxiliary/SentenceBreakProperty.txt"),271 ),272 )273}274275///|276async fn main {277 let generators : Array[(String, async (Generator) -> Unit)] = [278 ("ucd", generate_ucd),279 ("casefold", generate_casefold),280 ("idna", generate_idna),281 ("bidi", generate_bidi),282 ("segmentation", generate_segmentation),283 ]284 let command = @argparse.Command(285 "unicode-gen",286 about="Generate Unicode lookup tables with MoonBit",287 subcommands=[288 ..generators.map(entry => @argparse.Command(entry.0)),289 @argparse.Command("data", about="Run every generator"),290 @argparse.Command("all", about="Alias for `data`"),291 ],292 )293 let matches = command.parse() catch { error => fail(error.to_string()) }294 let selected = match matches.subcommand {295 Some((name, _)) if name != "data" && name != "all" =>296 generators.filter(entry => entry.0 == name)297 _ => generators298 }299 let g = Generator::new()300 for entry in selected {301 (entry.1)(g)302 }303 g.finish()304}305
///|
const UnicodeVersion : String = "18.0.0"
///|
/// One generator run: reads Unicode data files, each at most once, and writes
/// generated sources under the repository root.
priv struct Generator {
root : String
/// Contents of the data files read so far, by path under
/// `https://www.unicode.org/Public/<version>/`.
sources : Map[String, String]
/// Generated files, relative to `root`.
written : Array[String]
}
///|
async fn Generator::new() -> Generator {
let root = if @fs.exists("moon.work") && @fs.exists("tools/moon.mod") {
"."
} else if @fs.exists("../moon.work") && @fs.exists("moon.mod") {
".."
} else {
fail("Run the generator from the repository root or tools directory")
}
{ root, sources: Map([]), written: [], }
}
///|
/// The content of a data file, such as `ucd/UnicodeData.txt`, from the
/// download cache `tools/.cache/<version>/` or else from unicode.org.
async fn Generator::source(self : Generator, path : String) -> String {
if self.sources.get(path) is Some(content) {
return content
}
let name = match path.rev_split_once("/") {
Some((_, name)) => name.to_owned()
None => path
}
let cache_directory = "\{self.root}/tools/.cache/\{UnicodeVersion}"
let cache_path = "\{cache_directory}/\{name}"
let content = if @fs.exists(cache_path) {
println("Using cached \{name}")
@fs.read_text(cache_path)
} else {
println("Downloading \{name}...")
let url = "https://www.unicode.org/Public/\{UnicodeVersion}/\{path}"
let (response, body) = @http.get(url)
guard response.code >= 200 && response.code < 300 else {
fail("Download failed for \{url}: HTTP \{response.code}")
}
let content = body.text()
@fs.mkdir(
cache_directory,
permission=0o755,
recursive=true,
allow_existing=true,
)
@fs.write_text(cache_path, content, permission=0o644)
content
}
self.sources[path] = content
content
}
///|
/// The content of a file of the Unicode Character Database.
async fn Generator::ucd(self : Generator, path : String) -> String {
self.source("ucd/\{path}")
}
///|
async fn Generator::write(
self : Generator,
path : String,
source : String,
) -> Unit {
let full_path = "\{self.root}/\{path}"
if full_path.rev_split_once("/") is Some((directory, _)) {
@fs.mkdir(
directory.to_owned(),
permission=0o755,
recursive=true,
allow_existing=true,
)
}
@fs.write_text(full_path, source, permission=0o644)
self.written.push(path)
}
///|
async fn Generator::run_moon(
self : Generator,
arguments : Array[String],
) -> Unit {
let status = @shell.Cmd("moon", arguments, cwd=self.root).status()
guard status == 0 else { fail("moon \{arguments.join(" ")} failed") }
}
///|
/// Format the generated files and refresh the interfaces of the packages
/// holding them.
async fn Generator::finish(self : Generator) -> Unit {
self.run_moon(["fmt", ..self.written])
let packages : Array[String] = []
for path in self.written {
if path.rev_split_once("/") is Some((package_, _)) &&
!packages.contains(package_.to_owned()) {
packages.push(package_.to_owned())
}
}
self.run_moon(["info", ..packages])
}
///|
async fn generate_ucd(g : Generator) -> Unit {
let unicode_data = @ucd.parse_unicode_data(g.ucd("UnicodeData.txt"))
let exclusions = @ucd.parse_composition_exclusions(
g.ucd("CompositionExclusions.txt"),
)
let normalization_properties = g.ucd("DerivedNormalizationProps.txt")
g.write(
"normalization/normalization_data.mbt",
@ucd.emit_normalization_scan(unicode_data, normalization_properties),
)
g.write("ucd/ccc/ccc.mbt", @ucd.emit_ccc())
g.write(
"ucd/internal/normalization/normalization.mbt",
@ucd.emit_normalization(unicode_data, exclusions, normalization_properties),
)
g.write(
"ucd/decomposition/decomposition.mbt",
@ucd.emit_decomposition(unicode_data),
)
g.write(
"ucd/composition/composition.mbt",
@ucd.emit_composition(unicode_data, exclusions),
)
g.write("ucd/general_category/mark.mbt", @ucd.emit_marks(unicode_data))
g.write(
"ucd/general_category/general_category.mbt",
@ucd.emit_general_category(unicode_data),
)
g.write(
"ucd/punctuation/punctuation.mbt",
@ucd.emit_punctuation(unicode_data),
)
g.write(
"ucd/case_mapping/case_mapping.mbt",
@ucd.emit_case_mapping(
unicode_data,
@ucd.parse_special_casing(g.ucd("SpecialCasing.txt")),
),
)
g.write(
"ucd/case_mapping/case_context_data.mbt",
@ucd.emit_case_context(g.ucd("DerivedCoreProperties.txt")),
)
g.write("ucd/xid/xid.mbt", @ucd.emit_xid(g.ucd("DerivedCoreProperties.txt")))
// Each binary property lives in a package named after it.
for
entry in [
("Alphabetic", "DerivedCoreProperties.txt"),
("Lowercase", "DerivedCoreProperties.txt"),
("Uppercase", "DerivedCoreProperties.txt"),
("Default_Ignorable_Code_Point", "DerivedCoreProperties.txt"),
("Grapheme_Extend", "DerivedCoreProperties.txt"),
("White_Space", "PropList.txt"),
("Regional_Indicator", "PropList.txt"),
("Prepended_Concatenation_Mark", "PropList.txt"),
] {
let (property, source) = entry
let package_ = property.to_lower()
g.write(
"ucd/\{package_}/\{package_}.mbt",
@ucd.emit_binary_property(g.ucd(source), property~, source~),
)
}
g.write(
"ucd/emoji/emoji.mbt",
@ucd.emit_emoji(
g.ucd("emoji/emoji-data.txt"),
g.ucd("emoji/emoji-variation-sequences.txt"),
),
)
g.write(
"ucd/east_asian_width/east_asian_width.mbt",
@ucd.emit_east_asian_width(g.ucd("EastAsianWidth.txt")),
)
g.write(
"ucd/hangul_syllable_type/hangul_syllable_type.mbt",
@ucd.emit_hangul_syllable_type(g.ucd("HangulSyllableType.txt")),
)
g.write(
"ucd/joining_type/joining_type.mbt",
@ucd.emit_joining_type(g.ucd("extracted/DerivedJoiningType.txt")),
)
let joining_groups = g.ucd("extracted/DerivedJoiningGroup.txt")
g.write(
"ucd/joining_group/joining_group.mbt",
@ucd.emit_joining_group(joining_groups),
)
g.write("ucd/joining_group.mbt", @ucd.emit_joining_group_enum(joining_groups))
g.write(
"ucd/line_break/line_break.mbt",
@ucd.emit_line_break(g.ucd("LineBreak.txt")),
)
g.write("ucd/line_break.mbt", @ucd.emit_line_break_enum())
}
///|
async fn generate_casefold(g : Generator) -> Unit {
g.write(
"casefold/case_folding_data.mbt",
@ucd.emit_case_folding(g.ucd("CaseFolding.txt")),
)
g.write(
"casefold/nfkc_casefold_data.mbt",
@ucd.emit_nfkc_casefold(g.ucd("DerivedNormalizationProps.txt")),
)
}
///|
async fn generate_idna(g : Generator) -> Unit {
g.write(
"idna/internal/idna/mapping.mbt",
@idna_data.emit_mapping(g.source("idna/IdnaMappingTable.txt")),
)
g.write(
"idna/internal/idna/idna2008.mbt",
@idna_data.emit_idna2008(
g.source("idna/Idna2008.txt"),
g.ucd("Scripts.txt"),
),
)
}
///|
async fn generate_bidi(g : Generator) -> Unit {
g.write(
"bidi/bidi_class.mbt",
@bidi_data.emit_bidi_class(g.ucd("extracted/DerivedBidiClass.txt")),
)
g.write(
"bidi/internal/bidi/mirrored.mbt",
@bidi_data.emit_mirrored(g.ucd("BidiMirroring.txt")),
)
g.write(
"bidi/internal/bidi/bracket.mbt",
@bidi_data.emit_bracket(g.ucd("BidiBrackets.txt")),
)
}
///|
async fn generate_segmentation(g : Generator) -> Unit {
g.write(
"segmentation/grapheme_break_property.mbt",
@segmentation_data.emit_grapheme_break(
g.ucd("auxiliary/GraphemeBreakProperty.txt"),
g.ucd("emoji/emoji-data.txt"),
g.ucd("DerivedCoreProperties.txt"),
),
)
g.write(
"segmentation/word_break_property.mbt",
@segmentation_data.emit_word_break(g.ucd("auxiliary/WordBreakProperty.txt")),
)
g.write(
"segmentation/sentence_break_property.mbt",
@segmentation_data.emit_sentence_break(
g.ucd("auxiliary/SentenceBreakProperty.txt"),
),
)
}
///|
async fn main {
let generators : Array[(String, async (Generator) -> Unit)] = [
("ucd", generate_ucd),
("casefold", generate_casefold),
("idna", generate_idna),
("bidi", generate_bidi),
("segmentation", generate_segmentation),
]
let command = @argparse.Command(
"unicode-gen",
about="Generate Unicode lookup tables with MoonBit",
subcommands=[
..generators.map(entry => @argparse.Command(entry.0)),
@argparse.Command("data", about="Run every generator"),
@argparse.Command("all", about="Alias for `data`"),
],
)
let matches = command.parse() catch { error => fail(error.to_string()) }
let selected = match matches.subcommand {
Some((name, _)) if name != "data" && name != "all" =>
generators.filter(entry => entry.0 == name)
_ => generators
}
let g = Generator::new()
for entry in selected {
(entry.1)(g)
}
g.finish()
}