unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

idna/idna.mbt
1///|2/// IDNA (Internationalized Domain Names in Applications) processing3/// Implements UTS #46: Unicode IDNA Compatibility Processing4/// https://www.unicode.org/reports/tr46/56///|7/// Error types for IDNA processing8pub(all) suberror IdnaError {9  PunycodeError(@punycode.PunycodeError)10  DisallowedCodePoint(cp~ : Int32)11  InvalidLabel(String)12  InvalidHyphen13  LabelTooLong14  DomainTooLong15  EmptyLabel16  InvalidBidi17  InvalidJoiner18  InvalidContextO(cp~ : Int32)19  InvalidAcePrefix20  LeadingCombiningMark21}2223///|24pub extend IdnaError with Show::{to_string, output}2526///|27pub impl Show for IdnaError with fn output(self, logger) {28  match self {29    PunycodeError(error) =>30      logger.write_string("PunycodeError(\{error.to_string()})")31    DisallowedCodePoint(cp~) =>32      logger.write_string("DisallowedCodePoint(cp=\{cp.to_string()})")33    InvalidLabel(label) =>34      logger.write_string("InvalidLabel(\{label.to_string()})")35    InvalidHyphen => logger.write_string("InvalidHyphen")36    LabelTooLong => logger.write_string("LabelTooLong")37    DomainTooLong => logger.write_string("DomainTooLong")38    EmptyLabel => logger.write_string("EmptyLabel")39    InvalidBidi => logger.write_string("InvalidBidi")40    InvalidJoiner => logger.write_string("InvalidJoiner")41    InvalidContextO(cp~) =>42      logger.write_string("InvalidContextO(cp=\{cp.to_string()})")43    InvalidAcePrefix => logger.write_string("InvalidAcePrefix")44    LeadingCombiningMark => logger.write_string("LeadingCombiningMark")45  }46}4748///|49/// ACE prefix for Punycode labels50let ace_prefix : String = "xn--"5152///|53/// Converts a domain name to ASCII for DNS lookup.54///55/// Applies UTS #46 mapping and NFC normalization before validating Unicode56/// labels, including labels decoded from Punycode.57///58/// - `use_std3_ascii_rules` (default `true`): restricts ASCII label characters59///   to letters, digits, and hyphens.60/// - `check_hyphens` (default `true`): rejects leading or trailing hyphens and61///   labels with hyphens in both the third and fourth positions.62/// - `check_bidi` (default `true`): applies the RFC 5893 Bidi rule.63/// - `check_joiners` (default `true`): checks the RFC 5892 CONTEXTJ rules for64///   ZWNJ (U+200C) and ZWJ (U+200D).65/// - `check_idna2008` (default `false`): restricts Unicode label characters66///   to the IDNA2008 PVALID, CONTEXTJ, and CONTEXTO categories.67/// - `check_contexto` (default `false`): checks the RFC 5892 CONTEXTO rules.68/// - `verify_dns_length` (default `true`): checks ASCII label and domain lengths69///   against DNS limits.70pub fn to_ascii(71  domain : String,72  use_std3_ascii_rules~ : Bool = true,73  check_hyphens~ : Bool = true,74  check_bidi~ : Bool = true,75  check_joiners~ : Bool = true,76  check_idna2008~ : Bool = false,77  check_contexto~ : Bool = false,78  verify_dns_length~ : Bool = true,79) -> String raise IdnaError {80  if ascii_fast_path(81      domain,82      // IDNA2008 also limits plain ASCII labels to LDH after lowercasing.83      use_std3_ascii_rules=use_std3_ascii_rules || check_idna2008,84      check_hyphens~,85      verify_dns_length~,86    )87    is Some(ascii) {88    return ascii89  }90  to_ascii_general(91    domain,92    use_std3_ascii_rules~,93    check_hyphens~,94    check_bidi~,95    check_joiners~,96    check_idna2008~,97    check_contexto~,98    verify_dns_length~,99  )100}101102///|103/// ToASCII for any domain: map, normalize, then validate and encode each label.104fn to_ascii_general(105  domain : String,106  use_std3_ascii_rules~ : Bool,107  check_hyphens~ : Bool,108  check_bidi~ : Bool,109  check_joiners~ : Bool,110  check_idna2008~ : Bool,111  check_contexto~ : Bool,112  verify_dns_length~ : Bool,113) -> String raise IdnaError {114  // Map and normalize before splitting. Labels are validated below, after115  // existing A-labels have been decoded to their Unicode form. Mapping116  // leaves no ASCII uppercase letter, so every label is already lowercase.117  let processed = @normalization.nfc(map_domain(domain, use_std3_ascii_rules~))118  // When mapping and normalization leave the domain as it is, every119  // character has a valid status.120  let statuses_known = physical_equal(processed, domain)121  let length = processed.length()122  // Punycode is longer than the label it encodes; the size hint counts123  // bytes, two per UTF-16 code unit.124  let output = StringBuilder(capacity=4 * length)125  // The `validate_label` Bidi flags of every label so far, or-ed together.126  let mut domain_bidi = 0127  // The first label that the DNS length checks reject. Like the Bidi128  // error, it is reported only once every label has been validated.129  let mut dns_error : IdnaError? = None130  // Length of the output, which is all ASCII.131  let mut ascii_length = 0132  let mut start = 0133  for ;; {134    let end = label_end(processed, start)135    let label = processed[start:end]136    if start > 0 {137      output.write_char('.')138      ascii_length += 1139    }140    if label.length() == 0 {141      // Only the last label may be empty, after a dot (a trailing dot).142      // Any other empty label comes from ".." or from a label mapped to143      // nothing, such as a soft hyphen.144      guard end == length && start > 0 else { raise EmptyLabel }145      if dns_error is None {146        dns_error = Some(EmptyLabel)147      }148      break149    }150    let mut label_bidi = 0151    let label_length = if has_ace_prefix(label) {152      // Decode the A-label to validate its Unicode content.153      let decoded = decode_ace_label(label)154      let normalized = map_and_normalize_label(decoded, use_std3_ascii_rules~)155      let unicode_label = if normalized is Some(normalized) {156        normalized[:]157      } else {158        decoded[:]159      }160      label_bidi = validate_label(161        unicode_label,162        use_std3_ascii_rules~,163        check_hyphens~,164        check_joiners~,165        check_idna2008~,166        check_contexto~,167        check_bidi~,168        statuses_known=normalized is None,169      )170      // Encoding the Unicode form must give the A-label back (V7). The171      // decoded label always encodes back to an A-label in canonical form,172      // so it is encoded again only when mapping changed it.173      let round_trips = if normalized is None {174        is_canonical_ace_label(label)175      } else {176        encodes_to_ace_label(unicode_label, label)177      }178      guard round_trips else { raise InvalidAcePrefix }179      output.write_view(label)180      label.length()181    } else {182      // Raw A-labels must not be subjected to the hyphen checks before183      // decoding, because the xn-- prefix is intentional.184      label_bidi = validate_label(185        label,186        use_std3_ascii_rules~,187        check_hyphens~,188        check_joiners~,189        check_idna2008~,190        check_contexto~,191        check_bidi~,192        statuses_known~,193      )194      if is_all_ascii(label) {195        output.write_view(label)196        label.length()197      } else {198        output.write_string(ace_prefix)199        let encoded_length = @punycode.encode_to(label, output) catch {200          error => raise PunycodeError(error)201        }202        ace_prefix.length() + encoded_length203      }204    }205    domain_bidi = domain_bidi | label_bidi206    if dns_error is None && label_length > 63 {207      dns_error = Some(LabelTooLong)208    }209    ascii_length += label_length210    if end == length {211      break212    }213    start = end + 1214  }215  // A Bidi domain name, one with a right-to-left label, needs every label216  // to satisfy the Bidi rule.217  if domain_bidi == (RtlLabel | BreaksBidiRule) {218    raise InvalidBidi219  }220  if verify_dns_length {221    if dns_error is Some(error) {222      raise error223    }224    if ascii_length > 253 {225      raise DomainTooLong226    }227  }228  output.to_string()229}230231///|232/// Whether `label`, a lowercase A-label that decodes, is the encoding of its233/// decoding. Punycode gives each string a single lowercase encoding, and234/// decoding accepts only that one but for a delimiter with no basic code235/// point before it, which the encoding leaves out (see the Punycode fuzz236/// test "lowercase input that decodes encodes back to itself").237fn is_canonical_ace_label(label : StringView) -> Bool {238  let encoded = label[ace_prefix.length():]239  if encoded.length() == 0 || encoded[0] != '-' {240    return true241  }242  // The first delimiter is the last one only without basic code points.243  for i in 1..<encoded.length() {244    if encoded[i] == '-' {245      return true246    }247  }248  false249}250251///|252/// Whether encoding `unicode_label` gives back `label`, an A-label that is253/// already lowercase.254fn encodes_to_ace_label(255  unicode_label : StringView,256  label : StringView,257) -> Bool raise IdnaError {258  if is_all_ascii(unicode_label) {259    // An all-ASCII label is not encoded, so it is compared as it is.260    return unicode_label == label261  }262  let encoded = @punycode.encode(unicode_label) catch {263    error => raise PunycodeError(error)264  }265  label[ace_prefix.length():] == encoded[:]266}267268///|269/// Converts a domain name to Unicode for display.270///271/// Applies UTS #46 mapping and NFC normalization before validating Unicode272/// labels, including labels decoded from Punycode. DNS lengths are not checked.273///274/// - `use_std3_ascii_rules` (default `true`): restricts ASCII label characters275///   to letters, digits, and hyphens.276/// - `check_hyphens` (default `true`): rejects leading or trailing hyphens and277///   labels with hyphens in both the third and fourth positions.278/// - `check_bidi` (default `true`): applies the RFC 5893 Bidi rule.279/// - `check_joiners` (default `true`): checks the RFC 5892 CONTEXTJ rules for280///   ZWNJ (U+200C) and ZWJ (U+200D).281/// - `check_idna2008` (default `false`): restricts Unicode label characters282///   to the IDNA2008 PVALID, CONTEXTJ, and CONTEXTO categories.283/// - `check_contexto` (default `false`): checks the RFC 5892 CONTEXTO rules.284pub fn to_unicode(285  domain : String,286  use_std3_ascii_rules~ : Bool = true,287  check_hyphens~ : Bool = true,288  check_bidi~ : Bool = true,289  check_joiners~ : Bool = true,290  check_idna2008~ : Bool = false,291  check_contexto~ : Bool = false,292) -> String raise IdnaError {293  // ToUnicode does not verify DNS lengths.294  if ascii_fast_path(295      domain,296      use_std3_ascii_rules=use_std3_ascii_rules || check_idna2008,297      check_hyphens~,298      verify_dns_length=false,299    )300    is Some(unicode) {301    return unicode302  }303  to_unicode_general(304    domain,305    use_std3_ascii_rules~,306    check_hyphens~,307    check_bidi~,308    check_joiners~,309    check_idna2008~,310    check_contexto~,311  )312}313314///|315/// ToUnicode for any domain: decode, map, normalize and validate each label.316fn to_unicode_general(317  domain : String,318  use_std3_ascii_rules~ : Bool,319  check_hyphens~ : Bool,320  check_bidi~ : Bool,321  check_joiners~ : Bool,322  check_idna2008~ : Bool,323  check_contexto~ : Bool,324) -> String raise IdnaError {325  // Split into labels first (before mapping, to handle Punycode)326  let length = domain.length()327  // The size hint counts bytes, two per UTF-16 code unit.328  let output = StringBuilder(capacity=2 * length)329  let mut domain_bidi = 0330  let mut start = 0331  for ;; {332    let end = label_end(domain, start)333    let label = domain[start:end]334    if start > 0 {335      output.write_char('.')336    }337    if label.length() == 0 {338      // Only the last label may be empty, after a dot (a trailing dot).339      guard end == length && start > 0 else { raise EmptyLabel }340      break341    }342    let decoded = if has_ace_prefix(label) {343      decode_ace_label(label)[:]344    } else {345      label346    }347    let normalized = map_and_normalize_label(decoded, use_std3_ascii_rules~)348    let unicode_label = if normalized is Some(normalized) {349      normalized[:]350    } else {351      decoded352    }353    let label_bidi = validate_label(354      unicode_label,355      use_std3_ascii_rules~,356      check_hyphens~,357      check_joiners~,358      check_idna2008~,359      check_contexto~,360      check_bidi~,361      statuses_known=normalized is None,362    )363    domain_bidi = domain_bidi | label_bidi364    output.write_view(unicode_label)365    if end == length {366      break367    }368    start = end + 1369  }370  // A Bidi domain name, one with a right-to-left label, needs every label371  // to satisfy the Bidi rule.372  if domain_bidi == (RtlLabel | BreaksBidiRule) {373    raise InvalidBidi374  }375  output.to_string()376}377378///|379/// Decode an A-label and enforce the UTS #46 requirement that its decoded380/// form is non-empty and contains at least one non-ASCII code point.381fn decode_ace_label(label : StringView) -> String raise IdnaError {382  // The prefix is four ASCII characters, so four UTF-16 code units.383  let decoded = @punycode.decode(label[ace_prefix.length():]) catch {384    error => raise PunycodeError(error)385  }386  if is_all_ascii(decoded) {387    raise InvalidAcePrefix388  }389  decoded390}391392///|393fn is_all_ascii(label : StringView) -> Bool {394  label.all(c => c.to_int() < 0x80)395}396397///|398/// Check if a label has the ACE prefix "xn--", ignoring ASCII case399fn has_ace_prefix(label : StringView) -> Bool {400  label.length() >= 4 &&401  label[0] is ('x' | 'X') &&402  label[1] is ('n' | 'N') &&403  label[2] == '-' &&404  label[3] == '-'405}406