tools/segmentation_data/segmentation_data_wbtest.mbt1///|2test "packed values combine the three properties" {3 let values = parse_grapheme_break_values(4 (5 #|000D ; CR # Cc <control-000D>6 #|0300..0301 ; Extend # Mn [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT7 #|094D ; Extend # Mn DEVANAGARI SIGN VIRAMA8 #|9 ),10 (11 #|00A9 ; Extended_Pictographic# E0.6 [1] (©️) copyright12 #|0023 ; Emoji # E0.0 [1] (#️) hash sign13 #|14 ),15 (16 #|0041..0042 ; Alphabetic # L& [2] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER B17 #|094D ; InCB; Linker # Mn DEVANAGARI SIGN VIRAMA18 #|0915 ; InCB; Consonant # Lo DEVANAGARI LETTER KA19 #|0300..0301 ; InCB; Extend # Mn [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT20 #|21 ),22 )23 inspect(values.length(), content="1114112")24 debug_inspect(25 [0x0D, 0x23, 0x41, 0xA9, 0x300, 0x301, 0x302, 0x915, 0x94D].map(code_point => {26 values[code_point]27 }),28 content=(29 #|[1, 0, 0, 64, 36, 36, 0, 16, 52]30 ),31 )32}3334///|35test "grapheme break trie, high ranges, and constants" {36 let source = emit_grapheme_break(37 (38 #|000D ; CR # Cc <control-000D>39 #|E0001 ; Control # Cf LANGUAGE TAG40 #|1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z41 #|42 ),43 "",44 "",45 )46 for47 expected in [48 "let grapheme_break_trie_top : Bytes", "let grapheme_break_trie_mid : ReadOnlyArray[Int]",49 "let grapheme_break_trie_leaf : Bytes", "let grapheme_break_high_ranges : ReadOnlyArray[Int]",50 // Planes 4 and up: Other from U+40000, Control at U+E0001, Other again51 // from U+E0002.52 " 0x2000000,\n 0x7000083,\n 0x7000100,\n]", "const GcbCR : Int = 1", "const GcbRegionalIndicator : Int = 6",53 "const GcbLVT : Int = 13", "const InCBLinker : Int = 3", "const GcbMask : Int = 15",54 "const InCBShift : Int = 4", "const InCBMask : Int = 3", "const ExtendedPictographicBit : Int = 64",55 "fn grapheme_break_value(cp : Int) -> Int",56 ] {57 guard source.contains(expected) else {58 fail("Generated source is missing: \{expected}")59 }60 }61 // The default values need no constants.62 for unexpected in ["const GcbOther", "const InCBNone"] {63 guard !source.contains(unexpected) else {64 fail("Generated source should not contain: \{unexpected}")65 }66 }67}6869///|70test "unknown property values are rejected" {71 try emit_grapheme_break("0041; XX\n", "", "") catch {72 error =>73 inspect(74 error.to_string().contains("Unknown Grapheme_Cluster_Break: XX"),75 content="true",76 )77 } noraise {78 _ => fail("expected an unknown Grapheme_Cluster_Break to be rejected")79 }80 try emit_grapheme_break("", "", "0041; InCB; XX\n") catch {81 error =>82 inspect(83 error.to_string().contains("Unknown Indic_Conjunct_Break: XX"),84 content="true",85 )86 } noraise {87 _ => fail("expected an unknown Indic_Conjunct_Break to be rejected")88 }89}90
///|
test "packed values combine the three properties" {
let values = parse_grapheme_break_values(
(
#|000D ; CR # Cc <control-000D>
#|0300..0301 ; Extend # Mn [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT
#|094D ; Extend # Mn DEVANAGARI SIGN VIRAMA
#|
),
(
#|00A9 ; Extended_Pictographic# E0.6 [1] (©️) copyright
#|0023 ; Emoji # E0.0 [1] (#️) hash sign
#|
),
(
#|0041..0042 ; Alphabetic # L& [2] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER B
#|094D ; InCB; Linker # Mn DEVANAGARI SIGN VIRAMA
#|0915 ; InCB; Consonant # Lo DEVANAGARI LETTER KA
#|0300..0301 ; InCB; Extend # Mn [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT
#|
),
)
inspect(values.length(), content="1114112")
debug_inspect(
[0x0D, 0x23, 0x41, 0xA9, 0x300, 0x301, 0x302, 0x915, 0x94D].map(code_point => {
values[code_point]
}),
content=(
#|[1, 0, 0, 64, 36, 36, 0, 16, 52]
),
)
}
///|
test "grapheme break trie, high ranges, and constants" {
let source = emit_grapheme_break(
(
#|000D ; CR # Cc <control-000D>
#|E0001 ; Control # Cf LANGUAGE TAG
#|1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z
#|
),
"",
"",
)
for
expected in [
"let grapheme_break_trie_top : Bytes", "let grapheme_break_trie_mid : ReadOnlyArray[Int]",
"let grapheme_break_trie_leaf : Bytes", "let grapheme_break_high_ranges : ReadOnlyArray[Int]",
// Planes 4 and up: Other from U+40000, Control at U+E0001, Other again
// from U+E0002.
" 0x2000000,\n 0x7000083,\n 0x7000100,\n]", "const GcbCR : Int = 1", "const GcbRegionalIndicator : Int = 6",
"const GcbLVT : Int = 13", "const InCBLinker : Int = 3", "const GcbMask : Int = 15",
"const InCBShift : Int = 4", "const InCBMask : Int = 3", "const ExtendedPictographicBit : Int = 64",
"fn grapheme_break_value(cp : Int) -> Int",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
// The default values need no constants.
for unexpected in ["const GcbOther", "const InCBNone"] {
guard !source.contains(unexpected) else {
fail("Generated source should not contain: \{unexpected}")
}
}
}
///|
test "unknown property values are rejected" {
try emit_grapheme_break("0041; XX\n", "", "") catch {
error =>
inspect(
error.to_string().contains("Unknown Grapheme_Cluster_Break: XX"),
content="true",
)
} noraise {
_ => fail("expected an unknown Grapheme_Cluster_Break to be rejected")
}
try emit_grapheme_break("", "", "0041; InCB; XX\n") catch {
error =>
inspect(
error.to_string().contains("Unknown Indic_Conjunct_Break: XX"),
content="true",
)
} noraise {
_ => fail("expected an unknown Indic_Conjunct_Break to be rejected")
}
}