unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tests/support/support.mbt
1///|2/// Shared helpers for the workspace conformance test packages.3///4/// The official Unicode test data files live in the `data` directory of this5/// module. Tests read them at run time with `moonbitlang/core/async/fs`, so no6/// test code is generated from these files.78///|9/// Read a test data file by name, locating the module's `data` directory10/// relative to the current working directory.11pub async fn read_data_file(name : String) -> String {12  for prefix in ["tests/data/", "data/", "../data/", "../../data/"] {13    let path = "\{prefix}\{name}"14    if @fs.exists(path) {15      return @fs.read_text(path)16    }17  }18  fail(19    "test data file `\{name}` not found; run tests from the repository root or the tests module root",20  )21}2223///|24/// Parse a hexadecimal integer, e.g. `"1E0A"`.25pub fn parse_hex_int(text : StringView) -> Int32 raise {26  @string.parse_int(text.trim(), radix=16)27}2829///|30/// Parse a decimal integer.31pub fn parse_decimal_int(text : StringView) -> Int32 raise {32  @string.parse_int(text.trim(), radix=10)33}3435///|36/// Parse a space-separated sequence of hexadecimal code points.37pub fn parse_code_point_sequence(text : StringView) -> Array[Int32] raise {38  let result : Array[Int32] = []39  for part in text.trim().split(" ") {40    let part = part.trim()41    if !part.is_empty() {42      result.push(parse_hex_int(part))43    }44  }45  result46}4748///|49/// Build a string from code points. Returns `None` if any code point is a50/// surrogate, which MoonBit strings cannot represent.51pub fn code_points_to_string(code_points : ArrayView[Int32]) -> String? {52  let out = StringBuilder()53  for code_point in code_points {54    if code_point >= 0xD800 && code_point <= 0xDFFF {55      return None56    }57    out.write_char(code_point.unsafe_to_char())58  }59  Some(out.to_string())60}6162///|63/// Parse a space-separated list of bidi levels; `x`/`X` marks a removed64/// character and is represented as `-1`.65pub fn parse_levels(text : StringView) -> Array[Int32] raise {66  let levels : Array[Int32] = []67  for part in text.trim().split(" ") {68    let part = part.trim()69    if part.is_empty() {70      continue71    }72    match part {73      "x" | "X" => levels.push(-1)74      number => levels.push(parse_decimal_int(number))75    }76  }77  levels78}7980///|81/// Parse a space-separated list of visual ordering indices.82pub fn parse_reorder(text : StringView) -> Array[Int32] raise {83  let reorder : Array[Int32] = []84  for part in text.trim().split(" ") {85    let part = part.trim()86    if !part.is_empty() {87      reorder.push(parse_decimal_int(part))88    }89  }90  reorder91}9293///|94#derive(Debug, Eq)95pub struct ParsedText {96  text : String97  has_surrogate : Bool98}99100///|101pub extend ParsedText with Debug::{to_repr}102103///|104pub extend ParsedText with Eq::{not_equal, equal}105106///|107fn hex_value(character : Char) -> Int32? {108  match character {109    '0'..<='9' => Some(character.to_int() - '0'.to_int())110    'a'..<='f' => Some(character.to_int() - 'a'.to_int() + 10)111    'A'..<='F' => Some(character.to_int() - 'A'.to_int() + 10)112    _ => None113  }114}115116///|117fn parse_hex_chars(chars : ArrayView[Char]) -> Int32? {118  if chars.is_empty() {119    return None120  }121  let mut value = 0122  for character in chars {123    guard hex_value(character) is Some(digit) else { return None }124    value = value * 16 + digit125  }126  Some(value)127}128129///|130fn parse_braced_hex(chars : ArrayView[Char]) -> (Int32, ArrayView[Char])? {131  for remaining = chars, value = 0, has_digit = false {132    match remaining {133      ['}', .. rest] if has_digit => break Some((value, rest))134      [character, .. rest] => {135        guard hex_value(character) is Some(digit) else { break None }136        continue rest, value * 16 + digit, true137      }138      _ => break None139    }140  }141}142143///|144/// Unescape IdnaTestV2.txt text: literal characters plus `\uXXXX` and145/// `\x{...}` escapes. Tracks whether any surrogate code point appeared.146pub fn parse_escaped_text(text : StringView) -> ParsedText {147  if text == "\"\"" {148    return { text: "", has_surrogate: false, }149  }150  let chars = text.iter().collect()151  let out = StringBuilder()152  let mut has_surrogate = false153  for remaining = chars.exact_view() {154    match remaining {155      [] => break156      ['\\', 'u', a, b, c, d, .. rest] =>157        if parse_hex_chars([a, b, c, d]) is Some(code_point) {158          if code_point >= 0xD800 && code_point <= 0xDFFF {159            has_surrogate = true160          }161          out.write_char(code_point.unsafe_to_char())162          continue rest163        } else {164          out.write_char('\\')165          continue ['u', a, b, c, d, ..rest]166        }167      ['\\', 'x', '{', .. after_open] =>168        if parse_braced_hex(after_open) is Some((code_point, rest)) {169          if code_point >= 0xD800 && code_point <= 0xDFFF {170            has_surrogate = true171          }172          out.write_char(code_point.unsafe_to_char())173          continue rest174        } else {175          out.write_string("\\x{")176          continue after_open177        }178      [character, .. rest] => {179        out.write_char(character)180        continue rest181      }182    }183  }184  { text: out.to_string(), has_surrogate, }185}186187///|188/// Parse an IdnaTestV2.txt status list such as `[B1, C2]` or `[]`.189pub fn parse_status(text : StringView) -> Array[String]? {190  let text = text.trim()191  if text.is_empty() {192    return None193  }194  if text == "[]" {195    return Some([])196  }197  guard text.strip_prefix("[") is Some(after_open) else { return None }198  guard after_open.split_once("]") is Some((codes, _)) else { return None }199  let result : Array[String] = []200  for code in codes.replace_all(old=",", new=" ").split(" ") {201    let code = code.trim()202    if !code.is_empty() {203      result.push(code.to_owned())204    }205  }206  Some(result)207}208