unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/internal/parsing/parsing.mbt
1///|2#derive(Debug, Eq)3pub(all) struct CodePointRange {4  start : Int325  end : Int326}78///|9pub extend CodePointRange with Debug::{to_repr}1011///|12pub extend CodePointRange with Eq::{not_equal, equal}1314///|15pub fn strip_comment(line : StringView) -> StringView {16  if line.split_once("#") is Some((before, _)) {17    before.trim()18  } else {19    line.trim()20  }21}2223///|24/// The fields of every data line of a UCD file: the text before any `#`25/// comment, split at `;`, with every field trimmed. Blank and comment-only26/// lines are skipped.27pub fn data_lines(content : StringView) -> Iter[Array[StringView]] {28  content29  .split("\n")30  .filter_map(line => {31    let line = strip_comment(line)32    if line.is_empty() {33      None34    } else {35      Some(line.split(";").map(field => field.trim()).collect())36    }37  })38}3940///|41pub fn parse_hex_int(text : StringView) -> Int32 raise {42  @string.parse_int(text.trim(), radix=16)43}4445///|46pub fn parse_decimal_int(text : StringView) -> Int32 raise {47  @string.parse_int(text.trim(), radix=10)48}4950///|51pub fn parse_code_point_range(text : StringView) -> CodePointRange raise {52  let text = text.trim()53  if text.split_once("..") is Some((start_text, end_text)) {54    { start: parse_hex_int(start_text), end: parse_hex_int(end_text), }55  } else {56    let code_point = parse_hex_int(text)57    { start: code_point, end: code_point, }58  }59}6061///|62/// Read a UCD file for one enumerated property, whose data lines are63/// `range ; value`, and return the value of every code point.64///65/// Code points the file does not list take the value of the last66/// `# @missing: range; value` line covering them, or `default` when there is67/// none. Values are returned as written, so a `@missing` line that spells out68/// a long alias such as `Neutral` yields that alias.69pub fn parse_enumerated_property(70  content : StringView,71  default~ : String,72) -> FixedArray[String] raise {73  let values = FixedArray::make(@emitter.MaxCodePoint + 1, default)74  let fill = (range_text : StringView, value : StringView) => {75    let range = parse_code_point_range(range_text)76    let value = value.trim().to_owned()77    for code_point in range.start..<=range.end {78      values[code_point] = value79    }80  }81  for raw_line in content.split("\n") {82    if raw_line.trim().strip_prefix("# @missing:") is Some(declaration) &&83      declaration.split_once(";") is Some((range, value)) {84      fill(range, value)85    }86  }87  for fields in data_lines(content) {88    if fields is [range, value, ..] {89      fill(range, value)90    }91  }92  values93}9495///|96/// Read a UCD file for one enumerated property like97/// `parse_enumerated_property`, and convert the value of every code point98/// with `ordinal_of`, which accepts each spelling the file uses.99pub fn parse_enumerated_ordinals(100  content : StringView,101  property~ : String,102  default~ : String,103  ordinal_of : (StringView) -> Int32?,104) -> FixedArray[Int32] raise {105  let names = parse_enumerated_property(content, default~)106  let ordinals = FixedArray::make(names.length(), 0)107  for code_point, name in names {108    guard ordinal_of(name) is Some(ordinal) else {109      fail("Unknown \{property}: \{name}")110    }111    ordinals[code_point] = ordinal112  }113  ordinals114}115116///|117pub fn parse_code_point_sequence(text : StringView) -> Array[Int32] raise {118  let result : Array[Int32] = []119  for part in text.trim().split(" ") {120    let part = part.trim()121    if !part.is_empty() {122      result.push(parse_hex_int(part))123    }124  }125  result126}127