unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/normalization_test.mbt
1///|2test "normalization generator rejects incomplete Quick_Check data" {3  try4    @ucd.emit_normalization(5      @ucd.parse_unicode_data(""),6      [],7      "00C0 ; NFD_QC ; N",8    )9  catch {10    Failure::Failure(message) =>11      assert_true(message.contains("Missing Quick_Check properties"))12    error => raise error13  } noraise {14    _ => fail("expected incomplete Quick_Check data to fail")15  }16}1718///|19test "normalization generator rejects Maybe for decomposition forms" {20  for property in ["NFD_QC", "NFKD_QC"] {21    try22      @ucd.emit_normalization(23        @ucd.parse_unicode_data(""),24        [],25        "0300 ; \{property} ; M",26      )27    catch {28      Failure::Failure(message) =>29        assert_true(message.contains("Invalid Quick_Check value"))30      error => raise error31    } noraise {32      _ => fail("expected invalid Maybe value to fail")33    }34  }35}3637///|38test "normalization generator rejects overlapping ranges including explicit Yes" {39  let input =40    #|0300..0302 ; NFC_QC ; Y41    #|0301 ; NFC_QC ; M42  try @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input) catch {43    Failure::Failure(message) =>44      assert_true(message.contains("Overlapping Quick_Check ranges"))45    error => raise error46  } noraise {47    _ => fail("expected overlapping ranges to fail")48  }49}5051///|52test "normalization generator rejects out of range code points" {53  try54    @ucd.emit_normalization(55      @ucd.parse_unicode_data(""),56      [],57      "110000 ; NFC_QC ; N",58    )59  catch {60    Failure::Failure(message) =>61      assert_true(message.contains("Invalid Quick_Check range"))62    error => raise error63  } noraise {64    _ => fail("expected out of range code point to fail")65  }66}6768///|69test "normalization generator accepts explicit defaults and unrelated properties" {70  let input =71    #|# @missing: 0000..10FFFF; NFC_QC; Yes72    #|0041 ; NFC_QC ; Y73    #|0041 ; NFKC_QC ; Y74    #|0041 ; NFD_QC ; Y75    #|0041 ; NFKD_QC ; Y76    #|0042 ; Full_Composition_Exclusion77  let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)78  // An all-default table has a BMP block and an implicit supplementary tail.79  assert_true(source.contains("if cp >= 65536"))80}8182///|83test "normalization generator validates the full supplementary range" {84  // Force a stored value at both supplementary endpoints, including the last85  // slot of the last leaf. emit_normalization validates every intervening index.86  let input =87    #|10000 ; NFC_QC ; N88    #|10FFFF ; NFC_QC ; N89    #|10FFFF ; NFKC_QC ; N90    #|10FFFF ; NFD_QC ; N91    #|10FFFF ; NFKD_QC ; N92  let source = @ucd.emit_normalization(@ucd.parse_unicode_data(""), [], input)93  assert_true(source.contains("if cp >= 1114112"))94}9596///|97test "normalization generator rejects surrogate scalar properties" {98  let input =99    #|D800 ; NFC_QC ; N100    #|D800 ; NFKC_QC ; N101    #|D800 ; NFD_QC ; N102    #|D800 ; NFKD_QC ; N103  try @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), input) catch {104    Failure::Failure(message) =>105      assert_true(message.contains("Surrogate code points must have default"))106    error => raise error107  } noraise {108    _ => fail("expected surrogate scalar properties to fail")109  }110}111112///|113test "normalization scan generator covers empty and full supplementary tables" {114  let defaults =115    #|0041 ; NFC_QC ; Y116    #|0041 ; NFKC_QC ; Y117    #|0041 ; NFD_QC ; Y118    #|0041 ; NFKD_QC ; Y119  let empty = @ucd.emit_normalization_scan(120    @ucd.parse_unicode_data(""),121    defaults,122  )123  assert_true(empty.contains("NormalizationTailLeadIndex : Int = 0"))124  let full =125    #|10000 ; NFC_QC ; M126    #|10FFFF ; NFC_QC ; N127    #|10FFFF ; NFKC_QC ; N128    #|10FFFF ; NFD_QC ; N129    #|10FFFF ; NFKD_QC ; N130  let source = @ucd.emit_normalization_scan(@ucd.parse_unicode_data(""), full)131  assert_true(source.contains("NormalizationTailLeadIndex : Int = 1024"))132}133134///|135test "normalization scan generator reduces properties to one byte" {136  let quick_check =137    #|0041 ; NFC_QC ; M138    #|0041 ; NFKC_QC ; Y139    #|0041 ; NFD_QC ; Y140    #|0041 ; NFKD_QC ; N141  let source = @ucd.emit_normalization_scan(142    @ucd.parse_unicode_data("0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;"),143    quick_check,144  )145  // NFC_QC=Maybe is bit 9 and NFKD_QC=No is bit 13 of the packed properties;146  // both move down seven bits, leaving bit 0 for a nonzero CCC.147  assert_true(source.contains("  0,\n  68,\n  0,\n"))148  assert_true(source.contains("  0,\n  1,\n  0,\n"))149  // Leading surrogates fill whole blocks with their marker.150  assert_true(source.contains("  128,\n".repeat(32)))151  assert_true(152    source.contains("const NormalizationScanLeadSurrogate : Int = 128"),153  )154}155156///|157test "normalization scan generator gives flagged supplementary groups a slot" {158  let quick_check =159    #|0041 ; NFC_QC ; Y160    #|0041 ; NFKC_QC ; Y161    #|0041 ; NFD_QC ; Y162    #|10401 ; NFKD_QC ; N163  let source = @ucd.emit_normalization_scan(164    @ucd.parse_unicode_data(""),165    quick_check,166  )167  // Lead index 0 is clear and shares slot 0; lead index 1 gets slot 1.168  assert_true(169    source.contains("let normalization_lead_slots : Bytes = [\n  0,\n  1,\n]"),170  )171  // Two slots of 64 blocks; U+10401 is the second code point of slot 1.172  assert_true(173    source.contains("  0,\n".repeat(64) + "  1,\n" + "  0,\n".repeat(63) + "]"),174  )175  assert_true(source.contains("  0,\n".repeat(16) + "  0,\n  64,\n"))176}177