unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

idna/idna_test.mbt
1///|2/// Tests for IDNA processing (UTS #46)34///|5test "to_ascii: pure ASCII domain" {6  inspect(@idna.to_ascii("example.com"), content="example.com")7}89///|10test "to_ascii: simple IDN" {11  // münchen.de -> xn--mnchen-3ya.de12  inspect(@idna.to_ascii("münchen.de"), content="xn--mnchen-3ya.de")13}1415///|16test "to_ascii: accepts an existing valid A-label" {17  inspect(@idna.to_ascii("xn--mnchen-3ya.de"), content="xn--mnchen-3ya.de")18}1920///|21test "to_ascii: Chinese domain" {22  // 中文.com23  inspect(@idna.to_ascii("中文.com"), content="xn--fiq228c.com")24}2526///|27test "to_ascii: Japanese domain" {28  // ドメイン.jp29  inspect(@idna.to_ascii("ドメイン.jp"), content="xn--eckwd4c7c.jp")30}3132///|33test "to_ascii: mixed ASCII and Unicode" {34  inspect(35    @idna.to_ascii("bücher.example.com"),36    content="xn--bcher-kva.example.com",37  )38}3940///|41test "to_ascii: uppercase conversion" {42  inspect(@idna.to_ascii("EXAMPLE.COM"), content="example.com")43}4445///|46test "to_ascii: fullwidth separators" {47  // Fullwidth full stop U+FF0E should be treated as separator48  inspect(@idna.to_ascii("example\u{FF0E}com"), content="example.com")49}5051///|52test "to_unicode: pure ASCII domain" {53  inspect(@idna.to_unicode("example.com"), content="example.com")54}5556///|57test "to_unicode: Punycode domain" {58  // xn--mnchen-3ya.de -> münchen.de59  inspect(@idna.to_unicode("xn--mnchen-3ya.de"), content="münchen.de")60}6162///|63test "to_unicode: Chinese Punycode" {64  inspect(@idna.to_unicode("xn--fiq228c.com"), content="中文.com")65}6667///|68test "roundtrip: German" {69  let original = "münchen.de"70  let ascii = @idna.to_ascii(original)71  let back = @idna.to_unicode(ascii)72  assert_eq(back, original)73}7475///|76test "roundtrip: Chinese" {77  let original = "中文.com"78  let ascii = @idna.to_ascii(original)79  let back = @idna.to_unicode(ascii)80  assert_eq(back, original)81}8283///|84test "roundtrip: Japanese" {85  let original = "ドメイン.jp"86  let ascii = @idna.to_ascii(original)87  let back = @idna.to_unicode(ascii)88  assert_eq(back, original)89}9091///|92test "roundtrip: Korean" {93  let original = "한국어.kr"94  let ascii = @idna.to_ascii(original)95  let back = @idna.to_unicode(ascii)96  assert_eq(back, original)97}9899///|100test "roundtrip: Russian" {101  let original = "пример.ru"102  let ascii = @idna.to_ascii(original)103  let back = @idna.to_unicode(ascii)104  assert_eq(back, original)105}106107///|108test "invalid: disallowed character" {109  // U+0000 is disallowed110  @test.assert_raise(() => @idna.to_ascii("example\u{0000}.com"))111}112113///|114test "to_ascii: double dots handled" {115  // Double dots create an empty label which triggers EmptyLabel error116  let error = @test.expect_error(() => @idna.to_ascii("example..com"))117  guard error is @idna.EmptyLabel else { fail("Expected EmptyLabel") }118}119120///|121test "to_ascii: label too long" {122  // Create a label that will be > 63 characters after Punycode encoding123  // This is hard to test directly, so we test with already long ASCII124  let long_label = "a".repeat(64)125  @test.assert_raise(() => @idna.to_ascii(long_label + ".com"))126}127128///|129test "mapping: ignored characters" {130  // Soft hyphen U+00AD is ignored131  inspect(@idna.to_ascii("exam\u{00AD}ple.com"), content="example.com")132}133134///|135test "mapping: mapped characters" {136  // U+00DF (ß) maps to "ss"137  inspect(@idna.to_ascii("straße.de"), content="xn--strae-oqa.de")138}139140///|141test "validation: hyphen at start" {142  @test.assert_raise(() => @idna.to_ascii("-example.com"))143}144145///|146test "validation: hyphen at end" {147  @test.assert_raise(() => @idna.to_ascii("example-.com"))148}149150///|151test "std3 rules: space character rejected" {152  // Space U+0020 is rejected with STD3 rules (default) - not LDH153  @test.assert_raise(() => @idna.to_ascii("exam ple.com"))154}155156///|157test "std3 disabled: space character allowed" {158  // Space U+0020 is allowed without STD3 rules (marked "valid" in IDNA table)159  let result = @idna.to_ascii("exam ple.com", use_std3_ascii_rules=false)160  inspect(result, content="exam ple.com")161}162163///|164test "std3 disabled: allows some characters" {165  // Without STD3 rules, some characters are allowed166  let result = @idna.to_ascii(167    "example.com",168    use_std3_ascii_rules=false,169    check_hyphens=false,170  )171  inspect(result, content="example.com")172}173174///|175test "to_ascii: ideographic full stop" {176  // Ideographic full stop U+3002 should be treated as separator177  inspect(@idna.to_ascii("中文\u{3002}com"), content="xn--fiq228c.com")178}179180///|181test "to_ascii: multiple labels" {182  inspect(183    @idna.to_ascii("münchen.example.中文.com"),184    content="xn--mnchen-3ya.example.xn--fiq228c.com",185  )186}187188///|189test "to_unicode: mixed Punycode and ASCII" {190  inspect(191    @idna.to_unicode("xn--mnchen-3ya.example.com"),192    content="münchen.example.com",193  )194}195196///|197test "case folding: uppercase to lowercase" {198  inspect(@idna.to_ascii("MÜNCHEN.DE"), content="xn--mnchen-3ya.de")199}200201// =============================================================================202// Bidi validation tests (RFC 5893)203// =============================================================================204205///|206test "bidi: valid RTL label starting with Arabic letter" {207  // Arabic domain - starts with AL (Arabic Letter)208  // U+0645 (م) is Arabic Letter with bidi class AL209  let result = @idna.to_ascii("مثال.com", check_bidi=true)210  assert_true(result.has_prefix("xn--"))211}212213///|214test "bidi: valid RTL label with Hebrew" {215  // Hebrew domain - starts with R (Right-to-Left)216  // U+05D0 (א) is Hebrew with bidi class R217  let result = @idna.to_ascii("בדיקה.com", check_bidi=true)218  assert_true(result.has_prefix("xn--"))219}220221///|222test "bidi: RTL label cannot start with L character" {223  // Mix Hebrew (R) with leading Latin (L) - invalid224  // This label has RTL chars but doesn't start with R/AL225  let error = @test.expect_error(() => {226    @idna.to_ascii("aבדיקה.com", check_bidi=true)227  })228  guard error is @idna.InvalidBidi else { fail("Expected InvalidBidi") }229}230231///|232test "bidi: RTL label cannot contain L character" {233  // Hebrew label with Latin in the middle - L not allowed in RTL234  let error = @test.expect_error(() => {235    @idna.to_ascii("בדaיקה.com", check_bidi=true)236  })237  guard error is @idna.InvalidBidi else { fail("Expected InvalidBidi") }238}239240///|241test "bidi: RTL label cannot have both EN and AN" {242  // Arabic with both European Number and Arabic Number - invalid243  // U+0660-U+0669 are Arabic-Indic digits (AN)244  // 0-9 are European Numbers (EN)245  let error = @test.expect_error(() => {246    @idna.to_ascii("مثال1\u{0660}.com", check_bidi=true)247  })248  guard error is @idna.InvalidBidi else { fail("Expected InvalidBidi") }249}250251///|252test "bidi: LTR label cannot contain RTL characters" {253  // Latin label with Arabic character - invalid for LTR254  let error = @test.expect_error(() => {255    @idna.to_ascii("exampleم.com", check_bidi=true)256  })257  guard error is @idna.InvalidBidi else { fail("Expected InvalidBidi") }258}259260///|261test "bidi: pure ASCII label passes bidi check" {262  // Pure ASCII is LTR and valid263  let result = @idna.to_ascii("example.com", check_bidi=true)264  inspect(result, content="example.com")265}266267///|268test "bidi: to_unicode validates all labels at domain level" {269  let hebrew_label = @idna.to_ascii("בדיקה", check_bidi=false)270  let error = @test.expect_error(() => {271    @idna.to_unicode("\{hebrew_label}.123", check_bidi=true)272  })273  guard error is @idna.InvalidBidi else { fail("Expected InvalidBidi") }274}275276///|277test "bidi: disabled check allows mixed bidi" {278  // With bidi check disabled, mixed content is allowed279  let result = @idna.to_ascii("exampleم.com", check_bidi=false)280  assert_true(result.has_prefix("xn--"))281}282283// =============================================================================284// ContextJ validation tests (RFC 5892 - ZWNJ and ZWJ)285// =============================================================================286287///|288test "contextj: ZWNJ after Virama is valid" {289  // Devanagari Virama (U+094D) followed by ZWNJ (U+200C)290  // क् + ZWNJ - valid context291  ignore(292    @idna.to_ascii(293      "क\u{094D}\u{200C}ष.com",294      check_joiners=true,295      check_bidi=false,296    ),297  )298}299300///|301test "contextj: ZWNJ without valid context fails" {302  // ZWNJ at start of label - no valid context303  let error = @test.expect_error(() => {304    @idna.to_ascii("\u{200C}example.com", check_joiners=true)305  })306  guard error is @idna.InvalidJoiner else { fail("Expected InvalidJoiner") }307}308309///|310test "contextj: ZWNJ between non-joining chars fails" {311  // ZWNJ between Latin characters - invalid312  let error = @test.expect_error(() => {313    @idna.to_ascii("ex\u{200C}ample.com", check_joiners=true)314  })315  guard error is @idna.InvalidJoiner else { fail("Expected InvalidJoiner") }316}317318///|319test "contextj: ZWJ after Virama is valid" {320  // Devanagari Virama (U+094D) followed by ZWJ (U+200D)321  ignore(322    @idna.to_ascii(323      "क\u{094D}\u{200D}ष.com",324      check_joiners=true,325      check_bidi=false,326    ),327  )328}329330///|331test "contextj: ZWJ without Virama fails" {332  // ZWJ without preceding Virama - invalid333  let error = @test.expect_error(() => {334    @idna.to_ascii("ex\u{200D}ample.com", check_joiners=true)335  })336  guard error is @idna.InvalidJoiner else { fail("Expected InvalidJoiner") }337}338339///|340test "contextj: ZWJ at start of label fails" {341  // ZWJ at start - no preceding character342  let error = @test.expect_error(() => {343    @idna.to_ascii("\u{200D}example.com", check_joiners=true)344  })345  guard error is @idna.InvalidJoiner else { fail("Expected InvalidJoiner") }346}347348///|349test "contextj: disabled check allows joiners anywhere" {350  // With joiner check disabled, ZWNJ/ZWJ are allowed351  // ZWNJ is a Deviation character (not Ignored), so it stays in the string352  let result = @idna.to_ascii("ex\u{200C}ample.com", check_joiners=false)353  inspect(result, content="xn--example-9z6c.com")354}355356// =============================================================================357// ACE prefix validation tests358// =============================================================================359360///|361test "ace: valid Punycode decodes correctly" {362  // Test that a known Punycode domain decodes correctly363  let result = @idna.to_unicode("xn--mnchen-3ya.de")364  inspect(result, content="münchen.de")365}366367///|368test "ace: invalid Punycode after xn-- fails" {369  // Invalid Punycode sequence after xn--370  @test.assert_raise(() => @idna.to_unicode("xn--###invalid.com"))371}372373///|374test "ace: decoded A-label must contain non-ASCII code points" {375  let error = @test.expect_error(() => @idna.to_unicode("xn--example-.com"))376  guard error is @idna.InvalidAcePrefix else {377    fail("Expected InvalidAcePrefix")378  }379}380381///|382test "ace: uppercase XN-- prefix works" {383  // ACE prefix is case-insensitive384  let result = @idna.to_unicode("XN--MNCHEN-3YA.de")385  inspect(result, content="münchen.de")386}387388///|389test "ace: mixed case xN-- prefix works" {390  let result = @idna.to_unicode("xN--mnchen-3ya.de")391  inspect(result, content="münchen.de")392}393394// =============================================================================395// Leading combining mark tests396// =============================================================================397398///|399test "combining mark: at start of label fails" {400  // U+0301 (Combining Acute Accent) at start - invalid401  let error = @test.expect_error(() => @idna.to_ascii("\u{0301}example.com"))402  guard error is @idna.LeadingCombiningMark else {403    fail("Expected LeadingCombiningMark")404  }405}406407///|408test "combining mark: after base character is valid" {409  // e + combining acute = é (valid)410  let result = @idna.to_ascii("caf\u{0065}\u{0301}.com")411  // NFC normalizes to café412  assert_true(result.contains("caf"))413}414415///|416test "combining mark: multiple marks after base is valid" {417  // Base + multiple combining marks418  ignore(@idna.to_ascii("a\u{0301}\u{0327}.com", check_bidi=false))419}420421///|422test "combining mark: U+0300 grave at start fails" {423  // U+0300 (Combining Grave Accent) at start424  let error = @test.expect_error(() => @idna.to_ascii("\u{0300}test.com"))425  guard error is @idna.LeadingCombiningMark else {426    fail("Expected LeadingCombiningMark")427  }428}429430// =============================================================================431// Additional error condition tests432// =============================================================================433434///|435test "error: domain too long" {436  // Create a domain > 253 characters437  let long_label = "a".repeat(50)438  let domain = long_label +439    "." +440    long_label +441    "." +442    long_label +443    "." +444    long_label +445    "." +446    long_label +447    ".com"448  let error = @test.expect_error(() => @idna.to_ascii(domain))449  guard error is @idna.DomainTooLong else { fail("Expected DomainTooLong") }450}451452///|453test "error: empty domain" {454  let error = @test.expect_error(() => @idna.to_ascii(""))455  guard error is @idna.EmptyLabel else { fail("Expected EmptyLabel") }456}457458///|459test "error: only separator" {460  // Just a dot creates empty labels461  let error = @test.expect_error(() => @idna.to_ascii("."))462  guard error is @idna.EmptyLabel else { fail("Expected EmptyLabel") }463}464465///|466test "error: trailing dot creates empty label" {467  let error = @test.expect_error(() => @idna.to_ascii("example.com."))468  guard error is @idna.EmptyLabel else { fail("Expected EmptyLabel") }469}470471///|472test "error: hyphens at positions 3 and 4" {473  // "ab--cd" has hyphens at positions 3 and 4 (like ACE prefix pattern)474  let error = @test.expect_error(() => {475    @idna.to_ascii("ab--cd.com", check_hyphens=true)476  })477  guard error is @idna.InvalidHyphen else { fail("Expected InvalidHyphen") }478}479480///|481test "error: disallowed replacement character" {482  // U+FFFD (Replacement Character) is disallowed in IDNA483  @test.assert_raise(() => @idna.to_ascii("exam\u{FFFD}ple.com"))484}485486///|487test "error: disallowed private use character" {488  // U+E000 is private use area - disallowed489  @test.assert_raise(() => @idna.to_ascii("exam\u{E000}ple.com"))490}491492// =============================================================================493// Label separator tests494// =============================================================================495496///|497test "separator: halfwidth ideographic full stop" {498  // U+FF61 Halfwidth Ideographic Full Stop499  inspect(@idna.to_ascii("example\u{FF61}com"), content="example.com")500}501502///|503test "separator: all four separators" {504  // Test all four label separators in one domain505  // U+002E (.), U+3002 (。), U+FF0E (.), U+FF61 (。)506  let result = @idna.to_ascii(507    "a\u{002E}b\u{3002}c\u{FF0E}d\u{FF61}e",508    use_std3_ascii_rules=false,509    check_hyphens=false,510  )511  inspect(result, content="a.b.c.d.e")512}513514// =============================================================================515// Mapping edge cases516// =============================================================================517518///|519test "mapping: multiple ignored characters" {520  // Multiple soft hyphens should all be removed521  inspect(522    @idna.to_ascii(523      "e\u{00AD}x\u{00AD}a\u{00AD}m\u{00AD}p\u{00AD}l\u{00AD}e.com",524    ),525    content="example.com",526  )527}528529///|530test "mapping: fullwidth to halfwidth" {531  // Fullwidth Latin letters map to ASCII532  // U+FF21 (A) maps to U+0061 (a)533  inspect(@idna.to_ascii("\u{FF21}\u{FF22}\u{FF23}.com"), content="abc.com")534}535536///|537test "mapping: Greek case folding" {538  // Greek capital letters are case-folded to lowercase539  // ΛΟΓΟΣ -> λογοσ (all lowercase sigma, not final sigma)540  let result = @idna.to_ascii("ΛΟΓΟΣ.com", check_bidi=false)541  inspect(result, content="xn--oxapmbu.com")542}543544// =============================================================================545// Unicode script mixing tests546// =============================================================================547548///|549test "scripts: Cyrillic domain" {550  let result = @idna.to_ascii("москва.рф")551  assert_true(result.has_prefix("xn--"))552  assert_true(result.contains("."))553}554555///|556test "scripts: Thai domain" {557  let result = @idna.to_ascii("ไทย.th", check_bidi=false)558  assert_true(result.has_prefix("xn--"))559}560561///|562test "scripts: Arabic domain with digits" {563  // Arabic with Arabic-Indic digits564  let result = @idna.to_ascii(565    "مثال\u{0661}\u{0662}\u{0663}.com",566    check_bidi=false,567  )568  assert_true(result.has_prefix("xn--"))569}570571///|572test "ace: a delimiter with no basic code points is not canonical" {573  // "-mnchen-3ya" and "mnchen-3ya" decode alike, but only the second is the574  // encoding of the result.575  inspect(@idna.to_ascii("xn--mnchen-3ya.de"), content="xn--mnchen-3ya.de")576  inspect(@idna.to_unicode("xn--4ca.de"), content="ä.de")577  let error = @test.expect_error(() => @idna.to_ascii("xn---4ca.de"))578  guard error is @idna.InvalidAcePrefix else {579    fail("Expected InvalidAcePrefix")580  }581}582