tools/idna_data/idna_data.mbt1///|2/// Number of low bits holding the status code in a packed trie value or range.3const IdnaStatusBits : Int32 = 345///|6/// Status code of code points the mapping table does not list.7const IdnaDisallowed : Int32 = 489///|10priv struct MappingEntry {11 start : Int3212 end : Int3213 status : String14 mapping : Array[Int32]15}1617///|18fn parse_mapping_table(content : String) -> Array[MappingEntry] raise {19 let entries : Array[MappingEntry] = []20 for fields in @parsing.data_lines(content) {21 guard fields is [range, status, .. rest] else { continue }22 let range = @parsing.parse_code_point_range(range)23 let status = status.to_owned().to_lower().replace_all(old=" ", new="_")24 let mapping = if rest is [mapping, ..] {25 @parsing.parse_code_point_sequence(mapping)26 } else {27 []28 }29 entries.push({ start: range.start, end: range.end, status, mapping, })30 }31 entries32}3334///|35/// IDNA status values in the order of the `IdnaStatus` enum: the36/// constructor, the status IdnaMappingTable.txt writes (in lowercase, with37/// underscores for spaces), and a description.38let idna_statuses : ReadOnlyArray[(String, String, String)] = [39 ("Valid", "valid", "Valid character"),40 ("Ignored", "ignored", "Character is removed"),41 ("Mapped", "mapped", "Character is replaced by its mapping"),42 ("Deviation", "deviation", "Treated as valid in nontransitional mode"),43 ("Disallowed", "disallowed", "Character is not allowed"),44 (45 "DisallowedSTD3Valid", "disallowed_std3_valid", "Valid only without UseSTD3ASCIIRules",46 ),47 (48 "DisallowedSTD3Mapped", "disallowed_std3_mapped", "Mapped only without UseSTD3ASCIIRules",49 ),50]5152///|53/// Status code of a status: its index in `idna_statuses`. Unknown statuses54/// are disallowed.55fn status_code(status : String) -> Int32 {56 idna_statuses.search_by(entry => entry.1 == status).unwrap_or(IdnaDisallowed)57}5859///|60/// Emit the IDNA mapping table as a value trie.61///62/// Every code point has a packed value `mapping << 3 | status`, where63/// `mapping` numbers the distinct mappings (0 is the empty mapping). The64/// higher planes, which the trie does not cover, hold statuses without65/// mappings.66pub fn emit_mapping(content : String) -> String raise {67 // Mapping `i` is `mapping_data[mapping_offsets[i]:mapping_offsets[i + 1]]`.68 let mapping_ids : Map[Array[Int32], Int32] = Map([([], 0)])69 let mapping_offsets : Array[Int32] = [0, 0]70 let mapping_data : Array[Int32] = []71 let values = FixedArray::make(@emitter.MaxCodePoint + 1, IdnaDisallowed)72 for entry in parse_mapping_table(content) {73 let mapping = match mapping_ids.get(entry.mapping) {74 Some(id) => id75 None => {76 let id = mapping_ids.length()77 mapping_ids[entry.mapping] = id78 mapping_data.append(entry.mapping)79 mapping_offsets.push(mapping_data.length())80 id81 }82 }83 let value = (mapping << IdnaStatusBits) | status_code(entry.status)84 for code_point in entry.start..<=entry.end {85 values[code_point] = value86 }87 }88 let status_mask = (1 << IdnaStatusBits) - 189 let out = @emitter.Emitter::new()90 out.write_file_header("IDNA Mapping Table (UTS #46)", "IdnaMappingTable.txt")91 out.write_enum(92 type_name="IdnaStatus",93 doc=["IDNA status values"],94 converter="idna_status_from_ordinal",95 values=idna_statuses96 .iter()97 .map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })98 .collect(),99 )100 out.write_value_trie(101 property="IDNA",102 prefix="idna",103 lookup="fn lookup_idna_value",104 lookup_doc=["Look up the packed mapping table entry of a character."],105 value_bits=16,106 range_value_bits=IdnaStatusBits,107 values,108 )109 out.write_int_array(110 "Mapping `i` is `idna_mapping_data[idna_mapping_offsets[i]:idna_mapping_offsets[i + 1]]`",111 "idna_mapping_offsets",112 mapping_offsets,113 hex=false,114 )115 out.write_int_array(116 "Mapping data (flattened code points)",117 "idna_mapping_data",118 mapping_data,119 hex=true,120 )121 out.write(122 (123 $|///|124 $|/// Mapping table entry of a character, packed as125 $|/// `mapping << \{IdnaStatusBits} | status`. Looking one up allocates nothing.126 $|struct IdnaEntry(Int)127 $|128 $|///|129 $|/// Look up the mapping table entry of a character.130 $|pub fn lookup_idna_entry(c : Char) -> IdnaEntry {131 $| IdnaEntry(lookup_idna_value(c))132 $|}133 $|134 $|///|135 $|pub fn IdnaEntry::status(self : IdnaEntry) -> IdnaStatus {136 $| idna_status_from_ordinal(self.0 & \{status_mask})137 $|}138 $|139 $|///|140 $|/// Append the entry's mapping, which is empty unless the status is141 $|/// `Mapped`, `Deviation` or `DisallowedSTD3Mapped`.142 $|pub fn IdnaEntry::write_mapping(self : IdnaEntry, output : StringBuilder) -> Unit {143 $| let index = self.0 >> \{IdnaStatusBits}144 $| for code_point in idna_mapping_data.exact_view(145 $| start=idna_mapping_offsets[index],146 $| end=idna_mapping_offsets[index + 1],147 $| ) {148 $| output.write_char(code_point.unsafe_to_char())149 $| }150 $|}151 $|152 ),153 )154 out.finish()155}156
///|
/// Number of low bits holding the status code in a packed trie value or range.
const IdnaStatusBits : Int32 = 3
///|
/// Status code of code points the mapping table does not list.
const IdnaDisallowed : Int32 = 4
///|
priv struct MappingEntry {
start : Int32
end : Int32
status : String
mapping : Array[Int32]
}
///|
fn parse_mapping_table(content : String) -> Array[MappingEntry] raise {
let entries : Array[MappingEntry] = []
for fields in @parsing.data_lines(content) {
guard fields is [range, status, .. rest] else { continue }
let range = @parsing.parse_code_point_range(range)
let status = status.to_owned().to_lower().replace_all(old=" ", new="_")
let mapping = if rest is [mapping, ..] {
@parsing.parse_code_point_sequence(mapping)
} else {
[]
}
entries.push({ start: range.start, end: range.end, status, mapping, })
}
entries
}
///|
/// IDNA status values in the order of the `IdnaStatus` enum: the
/// constructor, the status IdnaMappingTable.txt writes (in lowercase, with
/// underscores for spaces), and a description.
let idna_statuses : ReadOnlyArray[(String, String, String)] = [
("Valid", "valid", "Valid character"),
("Ignored", "ignored", "Character is removed"),
("Mapped", "mapped", "Character is replaced by its mapping"),
("Deviation", "deviation", "Treated as valid in nontransitional mode"),
("Disallowed", "disallowed", "Character is not allowed"),
(
"DisallowedSTD3Valid", "disallowed_std3_valid", "Valid only without UseSTD3ASCIIRules",
),
(
"DisallowedSTD3Mapped", "disallowed_std3_mapped", "Mapped only without UseSTD3ASCIIRules",
),
]
///|
/// Status code of a status: its index in `idna_statuses`. Unknown statuses
/// are disallowed.
fn status_code(status : String) -> Int32 {
idna_statuses.search_by(entry => entry.1 == status).unwrap_or(IdnaDisallowed)
}
///|
/// Emit the IDNA mapping table as a value trie.
///
/// Every code point has a packed value `mapping << 3 | status`, where
/// `mapping` numbers the distinct mappings (0 is the empty mapping). The
/// higher planes, which the trie does not cover, hold statuses without
/// mappings.
pub fn emit_mapping(content : String) -> String raise {
// Mapping `i` is `mapping_data[mapping_offsets[i]:mapping_offsets[i + 1]]`.
let mapping_ids : Map[Array[Int32], Int32] = Map([([], 0)])
let mapping_offsets : Array[Int32] = [0, 0]
let mapping_data : Array[Int32] = []
let values = FixedArray::make(@emitter.MaxCodePoint + 1, IdnaDisallowed)
for entry in parse_mapping_table(content) {
let mapping = match mapping_ids.get(entry.mapping) {
Some(id) => id
None => {
let id = mapping_ids.length()
mapping_ids[entry.mapping] = id
mapping_data.append(entry.mapping)
mapping_offsets.push(mapping_data.length())
id
}
}
let value = (mapping << IdnaStatusBits) | status_code(entry.status)
for code_point in entry.start..<=entry.end {
values[code_point] = value
}
}
let status_mask = (1 << IdnaStatusBits) - 1
let out = @emitter.Emitter::new()
out.write_file_header("IDNA Mapping Table (UTS #46)", "IdnaMappingTable.txt")
out.write_enum(
type_name="IdnaStatus",
doc=["IDNA status values"],
converter="idna_status_from_ordinal",
values=idna_statuses
.iter()
.map(entry => @emitter.EnumValue::{ name: entry.0, comment: entry.2, })
.collect(),
)
out.write_value_trie(
property="IDNA",
prefix="idna",
lookup="fn lookup_idna_value",
lookup_doc=["Look up the packed mapping table entry of a character."],
value_bits=16,
range_value_bits=IdnaStatusBits,
values,
)
out.write_int_array(
"Mapping `i` is `idna_mapping_data[idna_mapping_offsets[i]:idna_mapping_offsets[i + 1]]`",
"idna_mapping_offsets",
mapping_offsets,
hex=false,
)
out.write_int_array(
"Mapping data (flattened code points)",
"idna_mapping_data",
mapping_data,
hex=true,
)
out.write(
(
$|///|
$|/// Mapping table entry of a character, packed as
$|/// `mapping << \{IdnaStatusBits} | status`. Looking one up allocates nothing.
$|struct IdnaEntry(Int)
$|
$|///|
$|/// Look up the mapping table entry of a character.
$|pub fn lookup_idna_entry(c : Char) -> IdnaEntry {
$| IdnaEntry(lookup_idna_value(c))
$|}
$|
$|///|
$|pub fn IdnaEntry::status(self : IdnaEntry) -> IdnaStatus {
$| idna_status_from_ordinal(self.0 & \{status_mask})
$|}
$|
$|///|
$|/// Append the entry's mapping, which is empty unless the status is
$|/// `Mapped`, `Deviation` or `DisallowedSTD3Mapped`.
$|pub fn IdnaEntry::write_mapping(self : IdnaEntry, output : StringBuilder) -> Unit {
$| let index = self.0 >> \{IdnaStatusBits}
$| for code_point in idna_mapping_data.exact_view(
$| start=idna_mapping_offsets[index],
$| end=idna_mapping_offsets[index + 1],
$| ) {
$| output.write_char(code_point.unsafe_to_char())
$| }
$|}
$|
),
)
out.finish()
}