unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

punycode/punycode_test.mbt
1///|2/// Tests for Punycode encoding/decoding (RFC 3492)34///|5test "encode: ASCII passthrough" {6  inspect(@punycode.encode("abc"), content="abc-")7}89///|10test "encode: ASCII view of a longer string" {11  let text = "abc.bücher"12  inspect(@punycode.encode(text[0:3]), content="abc-")13  inspect(@punycode.encode(text[3:3]), content="")14}1516///|17test "encode_to: appends and returns the length written" {18  let output = StringBuilder()19  output.write_string("xn--")20  for input in ["abc", "", "bücher", "ü"] {21    let length = @punycode.encode_to(input, output)22    output.write_string("|\{length}|")23  }24  inspect(output.to_string(), content="xn--abc-|4||0|bcher-kva|9|tda|3|")25}2627///|28test "encode: single non-ASCII" {29  // ü (U+00FC) -> "bcher" with encoded suffix30  inspect(@punycode.encode("bücher"), content="bcher-kva")31}3233///|34test "encode: München" {35  inspect(@punycode.encode("München"), content="Mnchen-3ya")36}3738///|39test "encode: Chinese" {40  // 中文 -> simplified Chinese characters41  inspect(@punycode.encode("中文"), content="fiq228c")42}4344///|45test "encode: Japanese" {46  // ドメイン -> Japanese katakana for "domain"47  inspect(@punycode.encode("ドメイン"), content="eckwd4c7c")48}4950///|51test "encode: Arabic" {52  // مثال -> Arabic example53  inspect(@punycode.encode("مثال"), content="mgbh0fb")54}5556///|57test "encode: mixed ASCII and non-ASCII" {58  inspect(@punycode.encode("münchen"), content="mnchen-3ya")59}6061///|62test "encode: empty string" {63  inspect(@punycode.encode(""), content="")64}6566///|67test "encode: pure ASCII" {68  inspect(@punycode.encode("example"), content="example-")69}7071///|72test "decode: ASCII passthrough" {73  inspect(@punycode.decode("abc-"), content="abc")74}7576///|77test "decode: single non-ASCII" {78  // bücher encodes to "bcher-kva"79  inspect(@punycode.decode("bcher-kva"), content="bücher")80}8182///|83test "decode: München" {84  inspect(@punycode.decode("Mnchen-3ya"), content="München")85}8687///|88test "decode: Chinese" {89  inspect(@punycode.decode("fiq228c"), content="中文")90}9192///|93test "decode: Japanese" {94  inspect(@punycode.decode("eckwd4c7c"), content="ドメイン")95}9697///|98test "decode: Arabic" {99  inspect(@punycode.decode("mgbh0fb"), content="مثال")100}101102///|103test "decode: mixed ASCII and non-ASCII" {104  inspect(@punycode.decode("mnchen-3ya"), content="münchen")105}106107///|108test "decode: empty string" {109  inspect(@punycode.decode(""), content="")110}111112///|113test "decode: pure ASCII" {114  inspect(@punycode.decode("example-"), content="example")115}116117///|118test "decode: rejects code points outside Unicode scalar range" {119  try @punycode.decode("en32g") catch {120    BadInput => ()121    error => raise error122  } noraise {123    _ => fail("Expected BadInput")124  }125  try @punycode.decode("ib9b") catch {126    BadInput => ()127    error => raise error128  } noraise {129    _ => fail("Expected BadInput for surrogate code point")130  }131}132133///|134test "encode: detects delta multiplication overflow" {135  let ascii_prefix = "a".repeat(3857)136  let input = "\{ascii_prefix}\u{10FFFF}"137  try @punycode.encode(input) catch {138    Overflow => ()139    error => raise error140  } noraise {141    _ => fail("Expected Overflow")142  }143}144145///|146test "roundtrip: various strings" {147  let test_cases = [148    "hello", "münchen", "München", "中文", "日本語", "العربية", "한국어",149    "Привет", "",150  ]151  for tc in test_cases {152    let encoded = @punycode.encode(tc)153    let decoded = @punycode.decode(encoded)154    assert_eq(decoded, tc)155  }156}157158///|159test "RFC 3492 examples: A (Arabic)" {160  // Example from RFC 3492 Section 7.1161  let arabic = "\u{0644}\u{064A}\u{0647}\u{0645}\u{0627}\u{0628}\u{062A}\u{0643}\u{0644}\u{0645}\u{0648}\u{0634}\u{0639}\u{0631}\u{0628}\u{064A}\u{061F}"162  inspect(@punycode.encode(arabic), content="egbpdaj6bu4bxfgehfvwxn")163}164165///|166test "RFC 3492 examples: B (Chinese simplified)" {167  // 他们为什么不说中文168  let chinese = "\u{4ED6}\u{4EEC}\u{4E3A}\u{4EC0}\u{4E48}\u{4E0D}\u{8BF4}\u{4E2D}\u{6587}"169  inspect(@punycode.encode(chinese), content="ihqwcrb4cv8a8dqg056pqjye")170}171172///|173test "RFC 3492 examples: C (Chinese traditional)" {174  // 他們爲什麽不說中文175  let chinese = "\u{4ED6}\u{5011}\u{7232}\u{4EC0}\u{9EBD}\u{4E0D}\u{8AAA}\u{4E2D}\u{6587}"176  inspect(@punycode.encode(chinese), content="ihqwctvzc91f659drss3x8bo0yb")177}178179///|180test "RFC 3492 examples: I (Japanese)" {181  // なぜみんな日本語を話してくれないのか182  let japanese = "\u{306A}\u{305C}\u{307F}\u{3093}\u{306A}\u{65E5}\u{672C}\u{8A9E}\u{3092}\u{8A71}\u{3057}\u{3066}\u{304F}\u{308C}\u{306A}\u{3044}\u{306E}\u{304B}"183  inspect(184    @punycode.encode(japanese),185    content="n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa",186  )187}188189///|190test "RFC 3492 examples: J (Korean)" {191  // 세계의모든사람들이한국어를이해한다면얼마나좋을까192  let korean = "\u{C138}\u{ACC4}\u{C758}\u{BAA8}\u{B4E0}\u{C0AC}\u{B78C}\u{B4E4}\u{C774}\u{D55C}\u{AD6D}\u{C5B4}\u{B97C}\u{C774}\u{D574}\u{D55C}\u{B2E4}\u{BA74}\u{C5BC}\u{B9C8}\u{B098}\u{C88B}\u{C744}\u{AE4C}"193  inspect(194    @punycode.encode(korean),195    content="989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c",196  )197}198199///|200test "RFC 3492 examples: N (Spanish)" {201  // PorsijunopuedensimijhacarconelEspaol202  let spanish = "Porqu\u{00E9}nopuedensimplementehablarenEspa\u{00F1}ol"203  inspect(204    @punycode.encode(spanish),205    content="PorqunopuedensimplementehablarenEspaol-fmd56a",206  )207}208209///|210test "decode: rejects non-ASCII input" {211  for212    input in [213      "\u{00FC}", "a-\u{00FC}", "\u{00FC}-a", "bcher-kva\u{00FC}", "\u{1F600}", "a-\u{1F600}",214      "\u{1F600}-a",215    ] {216    try @punycode.decode(input) catch {217      error => inspect(error, content="BadInput")218    } noraise {219      _ => fail("expected BadInput for \{input}")220    }221  }222}223224///|225test "decode: delimiters" {226  // Only the last delimiter separates the basic code points.227  inspect(@punycode.decode("-"), content="")228  inspect(@punycode.decode("a-"), content="a")229  inspect(@punycode.decode("a-b-"), content="a-b")230  inspect(@punycode.decode("--"), content="-")231  // A leading delimiter is skipped: no basic code points precede it.232  inspect(@punycode.decode("-tda"), content="\u{00FC}")233}234235///|236test "roundtrip: output longer than the reserved size" {237  // Large code points take more digits than the encoder reserves up front.238  for239    input in [240      "\u{10FFFF}",241      "a\u{10FFFF}",242      "\u{10FFFF}".repeat(40),243      "\u{0080}\u{10FFFF}".repeat(40),244      "\u{00FC}".repeat(300) + "a".repeat(300),245    ] {246    assert_eq(@punycode.decode(@punycode.encode(input)), input)247  }248}249