unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/idna_data/idna2008.mbt
1///|2/// The character classes needed by IDNA2008 validation. Contextual rules3/// themselves are checked against the complete label, not in this table.4fn parse_idna2008_properties(5  idna2008 : String,6  scripts : String,7) -> FixedArray[Int32] raise {8  let values = @parsing.parse_enumerated_ordinals(9    idna2008,10    property="IDNA2008 category",11    default="UNASSIGNED",12    category => {13      match category {14        "PVALID" | "CONTEXTJ" | "CONTEXTO" => Some(1)15        "DISALLOWED" | "UNASSIGNED" => Some(0)16        _ => None17      }18    },19  )20  // These are Script values, not Script_Extensions: in particular, U+30FB21  // KATAKANA MIDDLE DOT must not satisfy its own CONTEXTO condition.22  for23    code_point, script in @parsing.parse_enumerated_property(24      scripts,25      default="Unknown",26    ) {27    let flag = match script {28      "Greek" => 229      "Hebrew" => 430      "Hiragana" | "Katakana" | "Han" => 831      _ => 032    }33    values[code_point] = values[code_point] | flag34  }35  values36}3738///|39/// Emit the IDNA2008 allowed character set and the Script predicates used40/// by RFC 5892 Appendix A's CONTEXTO rules in one shared value trie.41pub fn emit_idna2008(idna2008 : String, scripts : String) -> String raise {42  let out = @emitter.Emitter::new()43  out.write_file_header(44    "IDNA2008 character validity and CONTEXTO Script data", "Idna2008.txt and Scripts.txt",45  )46  out.write_value_trie(47    property="IDNA2008",48    prefix="idna2008",49    lookup="fn lookup_idna_property_bits",50    lookup_doc=[51      "Bits for IDNA2008 allowed characters (1), Greek (2), Hebrew (4),", "and Hiragana, Katakana or Han (8).",52    ],53    value_bits=4,54    parse_idna2008_properties(idna2008, scripts),55  )56  out.write(57    (58      $|///|59      $|/// IDNA2008 character validity and CONTEXTO Script properties.60      $|struct IdnaProperties(Int)61      $|62      $|///|63      $|/// Look up a character's IDNA properties once for all needed checks.64      $|pub fn lookup_idna_properties(c : Char) -> IdnaProperties {65      $|  IdnaProperties(lookup_idna_property_bits(c))66      $|}67      $|68    ),69  )70  for71    (name, flag, description) in [72      (73        "is_idna2008_allowed", 1, "Whether a character is PVALID, CONTEXTJ or CONTEXTO in IDNA2008.",74      ),75      ("is_greek", 2, "Whether a character has Script=Greek."),76      ("is_hebrew", 4, "Whether a character has Script=Hebrew."),77      (78        "is_japanese", 8, "Whether a character has Script=Hiragana, Katakana or Han.",79      ),80    ] {81    out.write(82      (83        $|///|84        $|/// \{description}85        $|pub fn IdnaProperties::\{name}(self : IdnaProperties) -> Bool {86        $|  (self.0 & \{flag}) != 087        $|}88        $|89      ),90    )91  }92  out.finish()93}94