unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

normalization/scan_data_wbtest.mbt
1///|2/// The scan flags the generator derives from packed properties.3fn expected_scan_flags(c : Char) -> Int32 {4  let properties = @ccc.lookup_normalization_properties(c)5  let quick_check = properties >> 8 << 16  if (properties & 255) != 0 {7    quick_check | 18  } else {9    quick_check10  }11}1213///|14// The scan tables are a second copy of data that unicode-data also carries.15// Check every code point, so a data update that regenerates only one of the16// two modules cannot go unnoticed.17test "BMP scan flags agree with the packed properties" {18  for cp in 0..<0x10000 {19    let block = normalization_scan_index[cp >> NormalizationScanShift].to_int()20    let flags = normalization_scan_flags[(block << NormalizationScanShift) |21    (cp & NormalizationScanBlockMask)].to_int()22    let expected = if cp >= 0xD800 && cp < 0xDC00 {23      NormalizationScanLeadSurrogate24    } else {25      expected_scan_flags(cp.unsafe_to_char())26    }27    guard flags == expected else {28      fail("scan flags \{flags} != \{expected} at code point \{cp}")29    }30  }31}3233///|34test "supplementary scan flags agree with the packed properties" {35  for lead in 0..<1024 {36    let generated = if lead < normalization_lead_flags.length() {37      normalization_lead_flags[lead].to_int()38    } else {39      040    }41    let mut summary = 042    for trail in 0..<1024 {43      let cp = 0x10000 + (lead << 10) + trail44      let expected = expected_scan_flags(cp.unsafe_to_char())45      let default_tail = lead > NormalizationTailLeadIndex ||46        (47          lead == NormalizationTailLeadIndex &&48          0xDC00 + trail >= NormalizationTailTrail49        )50      guard !default_tail || expected == 0 else {51        fail("code point \{cp} in the default tail has scan flags \{expected}")52      }53      summary = summary | expected54      // The scan reads a pair's own flags whenever its group summary is set.55      if generated != 0 {56        let slot = normalization_lead_slots[lead].to_int()57        let block = normalization_pair_index[(58          slot << (10 - NormalizationPairShift)59        ) |60        (trail >> NormalizationPairShift)].to_int()61        let flags = normalization_pair_flags[(block << NormalizationPairShift) |62        (trail & NormalizationPairBlockMask)].to_int()63        guard flags == expected else {64          fail("pair flags \{flags} != \{expected} at code point \{cp}")65        }66      }67    }68    guard generated == summary else {69      fail("lead flags \{generated} != \{summary} for lead index \{lead}")70    }71  }72}7374///|75/// The scan masks of NFC, NFKC, NFD and NFKD.76#cfg(any(target="native", target="wasm"))77fn form_scan_masks() -> Array[Int32] {78  [(8, 3), (10, 3), (12, 1), (13, 1)].map(form => {79    let (shift, mask) = form80    (255 | (mask << shift)) >> 781  })82}8384///|85// The block scan skips code units below a fixed limit without reading their86// flags, so a data update that lowers a limit must not go unnoticed.87#cfg(any(target="native", target="wasm"))88test "code units below the clean limit have no scan flags" {89  for scan_mask in form_scan_masks() {90    let limit = clean_unit_limit(scan_mask)91    for cp in 0..<=limit {92      let block = normalization_scan_index[cp >> NormalizationScanShift].to_int()93      let flags = normalization_scan_flags[(block << NormalizationScanShift) |94      (cp & NormalizationScanBlockMask)].to_int()95      let clean = (flags & (scan_mask | NormalizationScanLeadSurrogate)) == 096      guard clean == (cp < limit) else {97        fail("code unit \{cp} breaks the clean limit \{limit}")98      }99    }100  }101}102103///|104/// Reference for the block skip: step over blocks of eight code units from105/// `start` while a whole block is left and all its units are below `limit`.106#cfg(any(target="native", target="wasm"))107fn reference_skip_clean_blocks(108  s : String,109  start : Int32,110  limit : Int32,111) -> Int32 {112  let length = s.length()113  let mut offset = start114  while offset + 8 <= length {115    for i in offset..<(offset + 8) {116      if s.unsafe_get(i).to_int() >= limit {117        return offset118      }119    }120    offset += 8121  }122  offset123}124125///|126/// Check the block skip against the reference at every start offset of `s`,127/// with the limit of every form.128#cfg(any(target="native", target="wasm"))129fn check_skip_clean_blocks(s : String, source : String) -> Unit raise {130  let length = s.length()131  for scan_mask in form_scan_masks() {132    let limit = clean_unit_limit(scan_mask)133    for start in 0..<=length {134      let actual = skip_clean_blocks(s, start, length, scan_mask)135      let expected = reference_skip_clean_blocks(s, start, limit)136      guard actual == expected else {137        fail(138          "skip_clean_blocks(\{source}, \{start}) with limit \{limit} = \{actual}, expected \{expected}",139        )140      }141    }142  }143}144145///|146/// Build a string from raw UTF-16 code units, which lets a test spell an147/// unpaired surrogate that a literal cannot.148#cfg(any(target="native", target="wasm"))149fn unit_string(units : Array[Int32]) -> String {150  let out = StringBuilder()151  for unit in units {152    out.write_char(unit.unsafe_to_char())153  }154  out.to_string()155}156157///|158/// The next value of a small linear congruential generator, used to build159/// mixed text without depending on a random package.160#cfg(any(target="native", target="wasm"))161fn probe_random(state : Int32) -> Int32 {162  (state * 1103515245 + 12345) & 0x7FFFFFFF163}164165///|166#cfg(any(target="native", target="wasm"))167test "block skip agrees with the scan one code unit at a time" {168  check_skip_clean_blocks("", "empty")169  check_skip_clean_blocks("abcdefg", "seven ASCII")170  check_skip_clean_blocks("abcdefgh", "eight ASCII")171  check_skip_clean_blocks("abcdefghijklmnop", "sixteen ASCII")172  check_skip_clean_blocks("Hello, world! ".repeat(16), "repeated ASCII")173  check_skip_clean_blocks("世界hello世界", "mixed BMP and ASCII")174  check_skip_clean_blocks(175    "Crème brûlée, naïve façade ".repeat(8),176    "Latin-1",177  )178  check_skip_clean_blocks(179    unit_string([0x61, 0x62, 0xDC00, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69]),180    "surrogate inside ASCII",181  )182  check_skip_clean_blocks("ab😀cd".repeat(8), "supplementary inside ASCII")183  // Put the last code unit below each limit and the limit itself in every184  // lane of a block.185  for unit in [0x7F, 0x80, 0x9F, 0xA0, 0xBF, 0xC0, 0x2FF, 0x300, 0xFFFF] {186    for ascii in 0..<=20 {187      check_skip_clean_blocks(188        "a".repeat(ascii) + unit_string([unit]) + "b".repeat(40 - ascii),189        "code unit \{unit} inside ASCII runs",190      )191    }192  }193}194195///|196#cfg(any(target="native", target="wasm"))197test "block skip agrees on random mixed text" {198  let mut state = 20261013199  for _ in 0..<64 {200    let out = StringBuilder()201    state = probe_random(state)202    let count = 8 + state % 200203    for _ in 0..<count {204      state = probe_random(state)205      let pick = state % 100206      if pick < 60 {207        out.write_char((0x20 + state % 0x5F).unsafe_to_char())208      } else if pick < 80 {209        out.write_char((0x80 + state % 0x700).unsafe_to_char())210      } else if pick < 90 {211        out.write_char((0x800 + state % 0xD000).unsafe_to_char())212      } else {213        // A well-formed surrogate pair for a supplementary code point.214        let code_point = 0x10000 + state % 0x1000215        out.write_char(code_point.unsafe_to_char())216      }217    }218    check_skip_clean_blocks(out.to_string(), "random text")219  }220}221