unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

idna/idna2008_test.mbt
1///|2fn expect_idna2008_rejection(3  domain : String,4  expected_cp : Int32,5  check_contexto~ : Bool = false,6) -> Unit raise {7  for ascii in [true, false] {8    try {9      if ascii {10        @idna.to_ascii(11          domain,12          use_std3_ascii_rules=false,13          check_idna2008=true,14          check_contexto~,15        )16      } else {17        @idna.to_unicode(18          domain,19          use_std3_ascii_rules=false,20          check_idna2008=true,21          check_contexto~,22        )23      }24    } catch {25      @idna.DisallowedCodePoint(cp~) => @test.assert_eq(cp, expected_cp)26      error => raise error27    } noraise {28      _ => fail("Expected IDNA2008 repertoire rejection for \{domain}")29    }30  }31}3233///|34fn expect_contexto_rejection(35  domain : String,36  expected_cp? : Int32,37) -> Unit raise {38  for ascii in [true, false] {39    try {40      if ascii {41        @idna.to_ascii(domain, check_contexto=true, check_bidi=false)42      } else {43        @idna.to_unicode(domain, check_contexto=true, check_bidi=false)44      }45    } catch {46      @idna.InvalidContextO(cp~) =>47        if expected_cp is Some(expected) {48          @test.assert_eq(cp, expected)49        }50      error => raise error51    } noraise {52      _ => fail("Expected CONTEXTO rejection for \{domain}")53    }54  }55}5657///|58test "IDNA2008 repertoire: default preserves UTS46 symbols" {59  inspect(@idna.to_ascii("☕.example"), content="xn--53h.example")60  inspect(@idna.to_unicode("xn--53h.example"), content="☕.example")61  inspect(@idna.to_ascii("♥.example"), content="xn--g6h.example")62  inspect(@idna.to_unicode("xn--g6h.example"), content="♥.example")63  for64    (input, cp) in [65      ("☕.example", 0x2615),66      ("xn--53h.example", 0x2615),67      ("♥.example", 0x2665),68      ("xn--g6h.example", 0x2665),69    ] {70    expect_idna2008_rejection(input, cp)71  }72}7374///|75test "IDNA2008 repertoire: mapping and NFC still precede validation" {76  for input in ["BÜCHER.DE", "bücher.de", "bu\u{0308}cher.de"] {77    inspect(78      @idna.to_ascii(input, check_idna2008=true, check_contexto=true),79      content="xn--bcher-kva.de",80    )81    inspect(82      @idna.to_unicode(input, check_idna2008=true, check_contexto=true),83      content="bücher.de",84    )85  }86  inspect(87    @idna.to_ascii("EXAM\u{00AD}PLE.COM", check_idna2008=true),88    content="example.com",89  )90  inspect(91    @idna.to_unicode("EXAM\u{00AD}PLE.COM", check_idna2008=true),92    content="example.com",93  )94}9596///|97test "IDNA2008 repertoire: unchanged labels and decoded ACE are checked" {98  for99    input in [100      "☕.example", "☕.EXAMPLE", "xn--53h.example", "XN--53H.EXAMPLE",101    ] {102    expect_idna2008_rejection(input, 0x2615)103  }104  inspect(105    @idna.to_ascii("faß.de", check_idna2008=true),106    content="xn--fa-hia.de",107  )108  inspect(109    @idna.to_unicode("xn--fa-hia.de", check_idna2008=true),110    content="faß.de",111  )112}113114///|115test "IDNA2008 repertoire: ASCII fast path respects independent STD3 flag" {116  inspect(117    @idna.to_ascii("under_score.example", use_std3_ascii_rules=false),118    content="under_score.example",119  )120  inspect(121    @idna.to_unicode("under_score.example", use_std3_ascii_rules=false),122    content="under_score.example",123  )124  expect_idna2008_rejection("under_score.example", 0x005F)125  expect_idna2008_rejection("UNDER_SCORE.EXAMPLE", 0x005F)126  for input in ["example.com", "EXAMPLE.COM"] {127    inspect(@idna.to_ascii(input, check_idna2008=true), content="example.com")128    inspect(@idna.to_unicode(input, check_idna2008=true), content="example.com")129  }130}131132///|133test "CONTEXTO: valid contexts survive Unicode and ACE conversion" {134  for135    label in [136      "l·l", "\u{0375}α", "\u{0375}\u{1F00}", "א\u{05F3}", "א\u{05F4}", "・カ",137      "カ・", "・あ", "・中", "・\u{20000}", "\u{0660}\u{0669}", "\u{06F0}\u{06F9}",138      "l·l·l", "\u{0375}\u{0375}α", "א\u{05F3}\u{05F4}", "\u{20000}・l·l",139    ] {140    // Pure digit labels test CONTEXTO independently of RFC 5893.141    let ace = @idna.to_ascii(label, check_bidi=false)142    for input in [label, ace] {143      @test.assert_eq(144        @idna.to_ascii(145          input,146          check_idna2008=true,147          check_contexto=true,148          check_bidi=false,149        ),150        ace,151      )152      @test.assert_eq(153        @idna.to_unicode(154          input,155          check_idna2008=true,156          check_contexto=true,157          check_bidi=false,158        ),159        label,160      )161    }162  }163}164165///|166test "CONTEXTO: middle dot requires an immediate l on both sides" {167  for168    label in [169      "·", "·l", "l·", "a·l", "l·a", "l·\u{0301}l", "l··l", "l·\u{20000}",170      "\u{20000}·l",171    ] {172    expect_contexto_rejection(label, expected_cp=0x00B7)173    let ace = @idna.to_ascii(label, check_bidi=false)174    expect_contexto_rejection(ace, expected_cp=0x00B7)175  }176  inspect(177    @idna.to_unicode("L·L", check_idna2008=true, check_contexto=true),178    content="l·l",179  )180}181182///|183test "CONTEXTO: Greek numeral sign checks following Script not block" {184  // U+03E3 is Coptic despite belonging to the Greek and Coptic block.185  for186    label in [187      "\u{0375}", "\u{0375}a", "α\u{0375}", "\u{0375}\u{03E3}", "\u{0375}\u{0375}",188    ] {189    expect_contexto_rejection(label, expected_cp=0x0375)190    let ace = @idna.to_ascii(label, check_bidi=false)191    expect_contexto_rejection(ace, expected_cp=0x0375)192  }193  // A supplementary Greek Script character satisfies the neighbor rule,194  // independently of its exclusion from the IDNA2008 allowed character set.195  let label = "\u{0375}\u{1D200}"196  let ace = @idna.to_ascii(label, check_bidi=false)197  for input in [label, ace] {198    @test.assert_eq(199      @idna.to_ascii(input, check_contexto=true, check_bidi=false),200      ace,201    )202    @test.assert_eq(203      @idna.to_unicode(input, check_contexto=true, check_bidi=false),204      label,205    )206  }207}208209///|210test "CONTEXTO: Hebrew punctuation requires preceding Hebrew Script" {211  for212    (label, cp) in [213      ("\u{05F3}", 0x05F3),214      ("\u{05F4}", 0x05F4),215      ("\u{05F3}א", 0x05F3),216      ("a\u{05F3}", 0x05F3),217      ("a\u{05F4}", 0x05F4),218      ("\u{20000}\u{05F3}", 0x05F3),219      ("א\u{20000}\u{05F4}", 0x05F4),220    ] {221    expect_contexto_rejection(label, expected_cp=cp)222    let ace = @idna.to_ascii(label, check_bidi=false)223    expect_contexto_rejection(ace, expected_cp=cp)224  }225}226227///|228test "CONTEXTO: Katakana middle dot needs a qualifying Script in its label" {229  // U+30FC has Script=Common, even though its Script_Extensions are Japanese.230  for label in ["・", "a・", "・ー", "カ.a・", "a・.カ"] {231    expect_contexto_rejection(label, expected_cp=0x30FB)232    let ace = @idna.to_ascii(label, check_bidi=false)233    expect_contexto_rejection(ace, expected_cp=0x30FB)234  }235}236237///|238test "CONTEXTO: Arabic digit sets cannot mix within one label" {239  for label in ["\u{0660}\u{06F0}", "\u{06F9}\u{0669}"] {240    expect_contexto_rejection(label)241    let ace = @idna.to_ascii(label, check_bidi=false)242    expect_contexto_rejection(ace)243  }244  let separate_labels = "\u{0660}.\u{06F0}"245  let ace = @idna.to_ascii(246    separate_labels,247    check_idna2008=true,248    check_contexto=true,249    check_bidi=false,250  )251  @test.assert_eq(252    @idna.to_unicode(253      ace,254      check_idna2008=true,255      check_contexto=true,256      check_bidi=false,257    ),258    separate_labels,259  )260}261262///|263test "CONTEXTO: adjacent labels cannot supply immediate context" {264  for265    (domain, cp) in [266      ("l.·l", 0x00B7),267      ("l·.l", 0x00B7),268      ("\u{0375}.α", 0x0375),269      ("א.\u{05F3}", 0x05F3),270      ("א.\u{05F4}", 0x05F4),271    ] {272    expect_contexto_rejection(domain, expected_cp=cp)273  }274}275276///|277test "CONTEXTO: neighboring violations precede later and deferred errors" {278  for279    (label, cp) in [280      ("l·\u{05F3}", 0x00B7),281      ("\u{0375}·", 0x0375),282      ("・l·a", 0x00B7),283      ("・l·", 0x00B7),284      ("・\u{0375}", 0x0375),285      ("\u{0660}\u{0375}\u{06F0}", 0x0375),286      ("\u{0660}\u{06F0}\u{0375}", 0x06F0),287    ] {288    expect_contexto_rejection(label, expected_cp=cp)289    let ace = @idna.to_ascii(label, check_bidi=false)290    expect_contexto_rejection(ace, expected_cp=cp)291  }292}293294///|295test "IDNA2008 repertoire and CONTEXTO flags are independent" {296  inspect(@idna.to_unicode("a·b", check_idna2008=true), content="a·b")297  expect_contexto_rejection("a·b", expected_cp=0x00B7)298  inspect(299    @idna.to_ascii("☕.example", check_contexto=true),300    content="xn--53h.example",301  )302  inspect(303    @idna.to_unicode("xn--53h.example", check_contexto=true),304    content="☕.example",305  )306  // The label's character validation finishes before contextual rules run,307  // so the disallowed coffee character takes precedence over the middle dot.308  let label = "a·☕"309  let ace = @idna.to_ascii(label)310  for input in [label, ace] {311    expect_idna2008_rejection(input, 0x2615, check_contexto=true)312  }313  inspect(314    @idna.to_ascii("-example", check_idna2008=true, check_hyphens=false),315    content="-example",316  )317  inspect(318    @idna.to_unicode("-example", check_idna2008=true, check_hyphens=false),319    content="-example",320  )321}322323///|324test "IDNA2008 repertoire: CONTEXTJ still obeys check_joiners" {325  for ascii in [true, false] {326    try {327      if ascii {328        @idna.to_ascii("a\u{200C}b", check_idna2008=true)329      } else {330        @idna.to_unicode("a\u{200C}b", check_idna2008=true)331      }332    } catch {333      @idna.InvalidJoiner => ()334      error => raise error335    } noraise {336      _ => fail("Expected existing CONTEXTJ validation")337    }338  }339  let ace = @idna.to_ascii(340    "a\u{200C}b",341    check_idna2008=true,342    check_contexto=true,343    check_joiners=false,344  )345  inspect(346    @idna.to_unicode(347      ace,348      check_idna2008=true,349      check_contexto=true,350      check_joiners=false,351    ),352    content="a\u{200C}b",353  )354}355