unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/gen/main.mbt
1///|2const UnicodeVersion : String = "18.0.0"34///|5/// One generator run: reads Unicode data files, each at most once, and writes6/// generated sources under the repository root.7priv struct Generator {8  root : String9  /// Contents of the data files read so far, by path under10  /// `https://www.unicode.org/Public/<version>/`.11  sources : Map[String, String]12  /// Generated files, relative to `root`.13  written : Array[String]14}1516///|17async fn Generator::new() -> Generator {18  let root = if @fs.exists("moon.work") && @fs.exists("tools/moon.mod") {19    "."20  } else if @fs.exists("../moon.work") && @fs.exists("moon.mod") {21    ".."22  } else {23    fail("Run the generator from the repository root or tools directory")24  }25  { root, sources: Map([]), written: [], }26}2728///|29/// The content of a data file, such as `ucd/UnicodeData.txt`, from the30/// download cache `tools/.cache/<version>/` or else from unicode.org.31async fn Generator::source(self : Generator, path : String) -> String {32  if self.sources.get(path) is Some(content) {33    return content34  }35  let name = match path.rev_split_once("/") {36    Some((_, name)) => name.to_owned()37    None => path38  }39  let cache_directory = "\{self.root}/tools/.cache/\{UnicodeVersion}"40  let cache_path = "\{cache_directory}/\{name}"41  let content = if @fs.exists(cache_path) {42    println("Using cached \{name}")43    @fs.read_text(cache_path)44  } else {45    println("Downloading \{name}...")46    let url = "https://www.unicode.org/Public/\{UnicodeVersion}/\{path}"47    let (response, body) = @http.get(url)48    guard response.code >= 200 && response.code < 300 else {49      fail("Download failed for \{url}: HTTP \{response.code}")50    }51    let content = body.text()52    @fs.mkdir(53      cache_directory,54      permission=0o755,55      recursive=true,56      allow_existing=true,57    )58    @fs.write_text(cache_path, content, permission=0o644)59    content60  }61  self.sources[path] = content62  content63}6465///|66/// The content of a file of the Unicode Character Database.67async fn Generator::ucd(self : Generator, path : String) -> String {68  self.source("ucd/\{path}")69}7071///|72async fn Generator::write(73  self : Generator,74  path : String,75  source : String,76) -> Unit {77  let full_path = "\{self.root}/\{path}"78  if full_path.rev_split_once("/") is Some((directory, _)) {79    @fs.mkdir(80      directory.to_owned(),81      permission=0o755,82      recursive=true,83      allow_existing=true,84    )85  }86  @fs.write_text(full_path, source, permission=0o644)87  self.written.push(path)88}8990///|91async fn Generator::run_moon(92  self : Generator,93  arguments : Array[String],94) -> Unit {95  let status = @shell.Cmd("moon", arguments, cwd=self.root).status()96  guard status == 0 else { fail("moon \{arguments.join(" ")} failed") }97}9899///|100/// Format the generated files and refresh the interfaces of the packages101/// holding them.102async fn Generator::finish(self : Generator) -> Unit {103  self.run_moon(["fmt", ..self.written])104  let packages : Array[String] = []105  for path in self.written {106    if path.rev_split_once("/") is Some((package_, _)) &&107      !packages.contains(package_.to_owned()) {108      packages.push(package_.to_owned())109    }110  }111  self.run_moon(["info", ..packages])112}113114///|115async fn generate_ucd(g : Generator) -> Unit {116  let unicode_data = @ucd.parse_unicode_data(g.ucd("UnicodeData.txt"))117  let exclusions = @ucd.parse_composition_exclusions(118    g.ucd("CompositionExclusions.txt"),119  )120  let normalization_properties = g.ucd("DerivedNormalizationProps.txt")121  g.write(122    "normalization/normalization_data.mbt",123    @ucd.emit_normalization_scan(unicode_data, normalization_properties),124  )125  g.write("ucd/ccc/ccc.mbt", @ucd.emit_ccc())126  g.write(127    "ucd/internal/normalization/normalization.mbt",128    @ucd.emit_normalization(unicode_data, exclusions, normalization_properties),129  )130  g.write(131    "ucd/decomposition/decomposition.mbt",132    @ucd.emit_decomposition(unicode_data),133  )134  g.write(135    "ucd/composition/composition.mbt",136    @ucd.emit_composition(unicode_data, exclusions),137  )138  g.write("ucd/general_category/mark.mbt", @ucd.emit_marks(unicode_data))139  g.write(140    "ucd/general_category/general_category.mbt",141    @ucd.emit_general_category(unicode_data),142  )143  g.write(144    "ucd/punctuation/punctuation.mbt",145    @ucd.emit_punctuation(unicode_data),146  )147  g.write(148    "ucd/case_mapping/case_mapping.mbt",149    @ucd.emit_case_mapping(150      unicode_data,151      @ucd.parse_special_casing(g.ucd("SpecialCasing.txt")),152    ),153  )154  g.write(155    "ucd/case_mapping/case_context_data.mbt",156    @ucd.emit_case_context(g.ucd("DerivedCoreProperties.txt")),157  )158  g.write("ucd/xid/xid.mbt", @ucd.emit_xid(g.ucd("DerivedCoreProperties.txt")))159  // Each binary property lives in a package named after it.160  for161    entry in [162      ("Alphabetic", "DerivedCoreProperties.txt"),163      ("Lowercase", "DerivedCoreProperties.txt"),164      ("Uppercase", "DerivedCoreProperties.txt"),165      ("Default_Ignorable_Code_Point", "DerivedCoreProperties.txt"),166      ("Grapheme_Extend", "DerivedCoreProperties.txt"),167      ("White_Space", "PropList.txt"),168      ("Regional_Indicator", "PropList.txt"),169      ("Prepended_Concatenation_Mark", "PropList.txt"),170    ] {171    let (property, source) = entry172    let package_ = property.to_lower()173    g.write(174      "ucd/\{package_}/\{package_}.mbt",175      @ucd.emit_binary_property(g.ucd(source), property~, source~),176    )177  }178  g.write(179    "ucd/emoji/emoji.mbt",180    @ucd.emit_emoji(181      g.ucd("emoji/emoji-data.txt"),182      g.ucd("emoji/emoji-variation-sequences.txt"),183    ),184  )185  g.write(186    "ucd/east_asian_width/east_asian_width.mbt",187    @ucd.emit_east_asian_width(g.ucd("EastAsianWidth.txt")),188  )189  g.write(190    "ucd/hangul_syllable_type/hangul_syllable_type.mbt",191    @ucd.emit_hangul_syllable_type(g.ucd("HangulSyllableType.txt")),192  )193  g.write(194    "ucd/joining_type/joining_type.mbt",195    @ucd.emit_joining_type(g.ucd("extracted/DerivedJoiningType.txt")),196  )197  let joining_groups = g.ucd("extracted/DerivedJoiningGroup.txt")198  g.write(199    "ucd/joining_group/joining_group.mbt",200    @ucd.emit_joining_group(joining_groups),201  )202  g.write("ucd/joining_group.mbt", @ucd.emit_joining_group_enum(joining_groups))203  g.write(204    "ucd/line_break/line_break.mbt",205    @ucd.emit_line_break(g.ucd("LineBreak.txt")),206  )207  g.write("ucd/line_break.mbt", @ucd.emit_line_break_enum())208}209210///|211async fn generate_casefold(g : Generator) -> Unit {212  g.write(213    "casefold/case_folding_data.mbt",214    @ucd.emit_case_folding(g.ucd("CaseFolding.txt")),215  )216  g.write(217    "casefold/nfkc_casefold_data.mbt",218    @ucd.emit_nfkc_casefold(g.ucd("DerivedNormalizationProps.txt")),219  )220}221222///|223async fn generate_idna(g : Generator) -> Unit {224  g.write(225    "idna/internal/idna/mapping.mbt",226    @idna_data.emit_mapping(g.source("idna/IdnaMappingTable.txt")),227  )228  g.write(229    "idna/internal/idna/idna2008.mbt",230    @idna_data.emit_idna2008(231      g.source("idna/Idna2008.txt"),232      g.ucd("Scripts.txt"),233    ),234  )235}236237///|238async fn generate_bidi(g : Generator) -> Unit {239  g.write(240    "bidi/bidi_class.mbt",241    @bidi_data.emit_bidi_class(g.ucd("extracted/DerivedBidiClass.txt")),242  )243  g.write(244    "bidi/internal/bidi/mirrored.mbt",245    @bidi_data.emit_mirrored(g.ucd("BidiMirroring.txt")),246  )247  g.write(248    "bidi/internal/bidi/bracket.mbt",249    @bidi_data.emit_bracket(g.ucd("BidiBrackets.txt")),250  )251}252253///|254async fn generate_segmentation(g : Generator) -> Unit {255  g.write(256    "segmentation/grapheme_break_property.mbt",257    @segmentation_data.emit_grapheme_break(258      g.ucd("auxiliary/GraphemeBreakProperty.txt"),259      g.ucd("emoji/emoji-data.txt"),260      g.ucd("DerivedCoreProperties.txt"),261    ),262  )263  g.write(264    "segmentation/word_break_property.mbt",265    @segmentation_data.emit_word_break(g.ucd("auxiliary/WordBreakProperty.txt")),266  )267  g.write(268    "segmentation/sentence_break_property.mbt",269    @segmentation_data.emit_sentence_break(270      g.ucd("auxiliary/SentenceBreakProperty.txt"),271    ),272  )273}274275///|276async fn main {277  let generators : Array[(String, async (Generator) -> Unit)] = [278    ("ucd", generate_ucd),279    ("casefold", generate_casefold),280    ("idna", generate_idna),281    ("bidi", generate_bidi),282    ("segmentation", generate_segmentation),283  ]284  let command = @argparse.Command(285    "unicode-gen",286    about="Generate Unicode lookup tables with MoonBit",287    subcommands=[288      ..generators.map(entry => @argparse.Command(entry.0)),289      @argparse.Command("data", about="Run every generator"),290      @argparse.Command("all", about="Alias for `data`"),291    ],292  )293  let matches = command.parse() catch { error => fail(error.to_string()) }294  let selected = match matches.subcommand {295    Some((name, _)) if name != "data" && name != "all" =>296      generators.filter(entry => entry.0 == name)297    _ => generators298  }299  let g = Generator::new()300  for entry in selected {301    (entry.1)(g)302  }303  g.finish()304}305