unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

normalization/quick_check_test.mbt
1///|2test "quick check handles Yes No and contextual Maybe" {3  for4    form in [5      @normalization.NFC,6      @normalization.NFD,7      @normalization.NFKC,8      @normalization.NFKD,9    ] {10    assert_true(@normalization.is_normalized("", form))11    assert_true(@normalization.is_normalized("Hello 世界 😀", form))12    assert_true(@normalization.is_normalized("\u{0301}", form))13  }14  assert_true(@normalization.is_normalized("é", @normalization.NFC))15  assert_false(@normalization.is_normalized("é", @normalization.NFD))16  assert_false(@normalization.is_normalized("e\u{0301}", @normalization.NFC))17  // Maybe does not imply that a composition is possible in this context.18  assert_true(@normalization.is_normalized("x\u{0301}", @normalization.NFC))19  assert_true(20    @normalization.is_normalized("a\u{0305}\u{0301}", @normalization.NFC),21  )22  assert_false(23    @normalization.is_normalized("x\u{0301}ffi", @normalization.NFKC),24  )25}2627///|28test "quick check verifies order and resets at starters" {29  for30    form in [31      @normalization.NFC,32      @normalization.NFD,33      @normalization.NFKC,34      @normalization.NFKD,35    ] {36    // These non-composing marks individually have Quick_Check=Yes.37    assert_false(@normalization.is_normalized("\u{0315}\u{0316}", form))38    assert_eq(39      @normalization.normalize("\u{0315}\u{0316}", form),40      "\u{0316}\u{0315}",41    )42    assert_true(@normalization.is_normalized("\u{0315}x\u{0316}", form))43  }44}4546///|47test "spans retain precomposed starters and supplementary offsets" {48  assert_eq(49    @normalization.nfc("😀abcă\u{0328}世界😀e\u{0301}!"),50    "😀abcą\u{0306}世界😀é!",51  )52  assert_eq(@normalization.nfd("😀é|Å😀"), "😀e\u{0301}|A\u{030A}😀")53  assert_eq(@normalization.nfkc("😀A\u{030A}|ffi😀"), "😀Å|ffi😀")54  assert_eq(55    @normalization.nfkd("😀A\u{030A}|ffi😀"),56    "😀A\u{030A}|ffi😀",57  )58}5960///|61test "CCC zero Maybe characters do not split Hangul composition" {62  for form in [@normalization.NFC, @normalization.NFKC] {63    assert_eq(64      @normalization.normalize("😀각|각😀", form),65      "😀각|각😀",66    )67    assert_true(@normalization.is_normalized("ᅡ", form))68    assert_false(@normalization.is_normalized("가", form))69    assert_eq(70      @normalization.normalize("ᄀ\u{0301}ᅡ", form),71      "ᄀ\u{0301}ᅡ",72    )73  }74}7576///|77test "Unicode 16 composite Maybe characters keep their preceding context" {78  // Kirat Rai: AA + AI must decompose AI before composing with AA.79  for form in [@normalization.NFC, @normalization.NFKC] {80    assert_true(@normalization.is_normalized("\u{16D68}", form))81    assert_false(@normalization.is_normalized("\u{16D63}\u{16D68}", form))82    assert_eq(83      @normalization.normalize("😀\u{16D63}\u{16D68}|", form),84      "😀\u{16D6A}|",85    )86  }87}8889///|90test "local segments preserve short and long copied runs" {91  for gap_length in [0, 1, 15, 16, 17, 40] {92    let gap = "-".repeat(gap_length)93    let prefix = "😀prefix".repeat(8)94    let suffix = "suffix😀".repeat(8)95    let decomposed = "\{prefix}e\u{0301}\{gap}a\u{030A}\{suffix}"96    let composed = "\{prefix}é\{gap}å\{suffix}"97    for form in [@normalization.NFC, @normalization.NFKC] {98      assert_eq(@normalization.normalize(decomposed, form), composed)99    }100    for form in [@normalization.NFD, @normalization.NFKD] {101      assert_eq(@normalization.normalize(composed, form), decomposed)102    }103  }104}105106///|107test "local checking continues after unchanged Maybe segments" {108  let prefix = "x\u{0301}😀".repeat(32)109  for form in [@normalization.NFC, @normalization.NFKC] {110    assert_true(@normalization.is_normalized(prefix, form))111    for112      (tail, expected) in [113        ("e\u{0301}", "é"),114        ("ă\u{0328}", "ą\u{0306}"),115        ("\u{0315}\u{0316}", "\u{0316}\u{0315}"),116        ("각", "각"),117        ("\u{16D63}\u{16D68}", "\u{16D6A}"),118      ] {119      let input = prefix + tail120      assert_false(@normalization.is_normalized(input, form))121      assert_eq(@normalization.normalize(input, form), prefix + expected)122      // Also exercise a changed segment followed by unchanged Maybe segments.123      assert_eq(124        @normalization.normalize(tail + prefix, form),125        expected + prefix,126      )127    }128  }129}130131///|132test "scratch reuse after expansion composition and long combining runs" {133  let marks = "\u{0315}".repeat(64) + "\u{0316}".repeat(64)134  let ordered = "\u{0316}".repeat(64) + "\u{0315}".repeat(64)135  let input = "ffi|\{marks}|e\u{0301}|x\u{0301}|😀A\u{030A}"136  for137    (form, expected) in [138      (@normalization.NFC, "ffi|\{ordered}|é|x\u{0301}|😀A\u{030A}"),139      (@normalization.NFKC, "ffi|\{ordered}|é|x\u{0301}|😀Å"),140      (@normalization.NFD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),141      (@normalization.NFKD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),142    ] {143    assert_eq(@normalization.normalize(input, form), expected)144    assert_false(@normalization.is_normalized(input, form))145    assert_true(@normalization.is_normalized(expected, form))146  }147}148149///|150test "fast runs retain boundaries across ordered marks and surrogate pairs" {151  for152    prefix in [153      "",154      "ASCII",155      "世界",156      "😀",157      "\u{10000}",158      "\u{10FFFF}",159      "世界😀".repeat(16),160    ] {161    for162      (tail, expected) in [163        ("A\u{0316}\u{0301}", "Á\u{0316}"),164        ("\u{0316}\u{0315}", "\u{0316}\u{0315}"),165        ("\u{0315}\u{0316}", "\u{0316}\u{0315}"),166        ("\u{0315}😀\u{0316}", "\u{0315}😀\u{0316}"),167        ("\u{0315}A\u{0301}", "\u{0315}Á"),168        ("\u{16D63}\u{16D68}", "\u{16D6A}"),169      ] {170      let input = prefix + tail171      let normalized = prefix + expected172      assert_eq(@normalization.nfc(input), normalized)173      assert_eq(174        @normalization.is_normalized(input, @normalization.NFC),175        input == normalized,176      )177    }178  }179}180181///|182test "resume fast runs between suspect segments and at the end" {183  for184    gap in [185      "",186      "A",187      "AA",188      "AAA",189      "Hello world! ".repeat(256),190      "世界😀".repeat(64),191    ] {192    for193      (form, input_part, normalized_part) in [194        (@normalization.NFC, "e\u{0301}", "é"),195        (@normalization.NFC, "x\u{0301}", "x\u{0301}"),196        (@normalization.NFC, "각", "각"),197        (@normalization.NFC, "\u{16D63}\u{16D68}", "\u{16D6A}"),198        (@normalization.NFD, "é", "e\u{0301}"),199        (@normalization.NFKC, "ffiA\u{030A}", "ffiÅ"),200        (@normalization.NFKD, "ffiÅ", "ffiA\u{030A}"),201      ] {202      for tail in ["", "\u{0315}\u{0316}", "😀\u{0315}\u{0316}"] {203        let ordered_tail = if tail == "" {204          ""205        } else if tail == "\u{0315}\u{0316}" {206          "\u{0316}\u{0315}"207        } else {208          "😀\u{0316}\u{0315}"209        }210        // Separators prevent the expected pieces from interacting; the final211        // gap + marks also checks recovery of the last skipped starter.212        let input = input_part + "|" + gap + "|" + input_part + "|" + gap + tail213        let expected = normalized_part +214          "|" +215          gap +216          "|" +217          normalized_part +218          "|" +219          gap +220          ordered_tail221        assert_eq(@normalization.normalize(input, form), expected)222        assert_eq(@normalization.is_normalized(input, form), input == expected)223        assert_true(@normalization.is_normalized(expected, form))224      }225    }226  }227  // After an unchanged Maybe segment, recover A as the next composing starter.228  assert_eq(@normalization.nfc("x\u{0301}AAA\u{0301}"), "x\u{0301}AAÁ")229  assert_false(230    @normalization.is_normalized("x\u{0301}AAA\u{0301}", @normalization.NFC),231  )232}233234///|235test "surrogate summaries respect form masks and recover complete boundaries" {236  for gap in ["😀", "\u{10FFFD}", "\u{10000}", "\u{1D400}"] {237    for238      (form, changed, fixed, normalized_gap) in [239        (@normalization.NFC, "e\u{0301}", "é", gap),240        (@normalization.NFD, "é", "e\u{0301}", gap),241        (242          @normalization.NFKC,243          "e\u{0301}",244          "é",245          if gap == "\u{1D400}" {246            "A"247          } else {248            gap249          },250        ),251        (252          @normalization.NFKD,253          "é",254          "e\u{0301}",255          if gap == "\u{1D400}" {256            "A"257          } else {258            gap259          },260        ),261      ] {262      // No separator before the marks: recovery must retain the full pair,263      // including when a changed prefix has already allocated scratch storage.264      for prefix in ["", changed + "|"] {265        let fixed_prefix = if prefix == "" { "" } else { fixed + "|" }266        let input = prefix + gap.repeat(8) + "\u{0315}\u{0316}|" + changed267        let expected = fixed_prefix +268          normalized_gap.repeat(8) +269          "\u{0316}\u{0315}|" +270          fixed271        assert_eq(@normalization.normalize(input, form), expected)272        assert_false(@normalization.is_normalized(input, form))273        assert_true(@normalization.is_normalized(expected, form))274      }275    }276  }277  // Active groups still decode mappings, combining marks and backward composition.278  assert_eq(@normalization.nfd("\u{1D15E}"), "\u{1D157}\u{1D165}")279  assert_eq(@normalization.nfc("\u{11099}\u{110BA}"), "\u{1109A}")280  assert_eq(281    @normalization.nfc("😀\u{1D165}\u{1D167}"),282    "😀\u{1D167}\u{1D165}",283  )284}285286///|287test "surrogate summaries keep isolated code units unchanged" {288  for cp in [0xD800, 0xD835, 0xDBFF, 0xDC00, 0xDFFF] {289    let lone = String::from_array([cp.unsafe_to_char()])290    for input in [lone, lone + "A", "A" + lone] {291      for292        form in [293          @normalization.NFC,294          @normalization.NFD,295          @normalization.NFKC,296          @normalization.NFKD,297        ] {298        assert_eq(@normalization.normalize(input, form), input)299        assert_true(@normalization.is_normalized(input, form))300      }301    }302  }303}304305///|306test "unpaired surrogates stay clean next to suspect characters" {307  // U+D835 leads a group of 1024 code points that has flagged characters.308  let lead = String::from_array([(0xD835).unsafe_to_char()])309  let trail = String::from_array([(0xDC00).unsafe_to_char()])310  for311    (input, expected) in [312      (lead + "e\u{0301}", lead + "é"),313      ("e\u{0301}" + lead, "é" + lead),314      (trail + lead + "e\u{0301}", trail + lead + "é"),315      (lead + "\u{0301}", lead + "\u{0301}"),316      (lead + "\u{0315}\u{0316}", lead + "\u{0316}\u{0315}"),317      // The second unit pairs with the trailing one; the first stays alone.318      (319        lead + lead + trail + "\u{0315}\u{0316}",320        lead + lead + trail + "\u{0316}\u{0315}",321      ),322      (323        "e\u{0301}|" + lead + lead + trail + lead,324        "é|" + lead + lead + trail + lead,325      ),326    ] {327    assert_eq(@normalization.nfc(input), expected)328    assert_eq(329      @normalization.is_normalized(input, @normalization.NFC),330      input == expected,331    )332    assert_true(@normalization.is_normalized(expected, @normalization.NFC))333  }334}335336///|337test "clean pairs in flagged groups become the safe boundary" {338  // U+1D400 is clean for NFC and NFD, but shares its group with U+1D15E.339  let pair = "\u{1D400}"340  for prefix in ["", "世界", "e\u{0301}|"] {341    let fixed = if prefix == "e\u{0301}|" { "é|" } else { prefix }342    for count in [1, 2, 64] {343      let run = pair.repeat(count)344      // Marks attach to the last pair, so only they are reordered.345      assert_eq(346        @normalization.nfc(prefix + run + "\u{0315}\u{0316}"),347        fixed + run + "\u{0316}\u{0315}",348      )349      // A starter after the run composes without disturbing it.350      assert_eq(351        @normalization.nfc(prefix + run + "e\u{0301}"),352        fixed + run + "é",353      )354      assert_eq(355        @normalization.is_normalized(prefix + run, @normalization.NFC),356        prefix == fixed,357      )358    }359  }360}361362///|363test "block scans stop at every offset within and between blocks" {364  // Each case is a character that only some forms leave alone, with its365  // normalization in NFC, NFD, NFKC and NFKD.366  for367    (suspect, nfc, nfd, nfkc, nfkd) in [368      ("\u{A0}", "\u{A0}", "\u{A0}", " ", " "),369      ("é", "é", "e\u{0301}", "é", "e\u{0301}"),370      ("e\u{0301}", "é", "e\u{0301}", "é", "e\u{0301}"),371      ("😀", "😀", "😀", "😀", "😀"),372      (373        "\u{1D15E}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}",374        "\u{1D157}\u{1D165}",375      ),376    ] {377    for before in 0..<48 {378      for after in [0, 1, 7, 8, 9, 17] {379        let prefix = "a".repeat(before)380        let suffix = "z".repeat(after)381        let input = prefix + suspect + suffix382        for383          (form, expected) in [384            (@normalization.NFC, nfc),385            (@normalization.NFD, nfd),386            (@normalization.NFKC, nfkc),387            (@normalization.NFKD, nfkd),388          ] {389          let normalized = prefix + expected + suffix390          assert_eq(@normalization.normalize(input, form), normalized)391          assert_eq(392            @normalization.is_normalized(input, form),393            input == normalized,394          )395        }396      }397    }398  }399}400401///|402test "long ASCII runs at every length" {403  for404    form in [405      @normalization.NFC,406      @normalization.NFD,407      @normalization.NFKC,408      @normalization.NFKD,409    ] {410    for ascii in 0..<=40 {411      let run = "a".repeat(ascii)412      // These prefixes decompose under no form, so prefix + run is unchanged.413      for prefix in ["", "世界", "😀"] {414        assert_true(@normalization.is_normalized(prefix + run, form))415        assert_eq(@normalization.normalize(prefix + run, form), prefix + run)416      }417    }418  }419  for ascii in 0..<=40 {420    let run = "a".repeat(ascii)421    // A change before the run, after it, and on both sides: the run itself is422    // copied as it is either way.423    assert_eq(@normalization.nfc("e\u{0301}" + run), "é" + run)424    assert_false(425      @normalization.is_normalized("e\u{0301}" + run, @normalization.NFC),426    )427    assert_eq(428      @normalization.nfc("世界" + run + "e\u{0301}"),429      "世界" + run + "é",430    )431    assert_eq(@normalization.nfkc("ffi" + run), "ffi" + run)432    assert_eq(@normalization.nfc(run + "e\u{0301}"), run + "é")433    assert_false(434      @normalization.is_normalized(run + "e\u{0301}", @normalization.NFC),435    )436  }437}438