unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/segmentation_data/segmentation_data_wbtest.mbt
1///|2test "packed values combine the three properties" {3  let values = parse_grapheme_break_values(4    (5      #|000D          ; CR # Cc       <control-000D>6      #|0300..0301    ; Extend # Mn   [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT7      #|094D          ; Extend # Mn       DEVANAGARI SIGN VIRAMA8      #|9    ),10    (11      #|00A9          ; Extended_Pictographic# E0.6   [1] (©️)       copyright12      #|0023          ; Emoji                # E0.0   [1] (#️)       hash sign13      #|14    ),15    (16      #|0041..0042    ; Alphabetic # L&   [2] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER B17      #|094D          ; InCB; Linker # Mn       DEVANAGARI SIGN VIRAMA18      #|0915          ; InCB; Consonant # Lo       DEVANAGARI LETTER KA19      #|0300..0301    ; InCB; Extend # Mn   [2] COMBINING GRAVE ACCENT..COMBINING ACUTE ACCENT20      #|21    ),22  )23  inspect(values.length(), content="1114112")24  debug_inspect(25    [0x0D, 0x23, 0x41, 0xA9, 0x300, 0x301, 0x302, 0x915, 0x94D].map(code_point => {26      values[code_point]27    }),28    content=(29      #|[1, 0, 0, 64, 36, 36, 0, 16, 52]30    ),31  )32}3334///|35test "grapheme break trie, high ranges, and constants" {36  let source = emit_grapheme_break(37    (38      #|000D          ; CR # Cc       <control-000D>39      #|E0001         ; Control # Cf       LANGUAGE TAG40      #|1F1E6..1F1FF  ; Regional_Indicator # So  [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z41      #|42    ),43    "",44    "",45  )46  for47    expected in [48      "let grapheme_break_trie_top : Bytes", "let grapheme_break_trie_mid : ReadOnlyArray[Int]",49      "let grapheme_break_trie_leaf : Bytes", "let grapheme_break_high_ranges : ReadOnlyArray[Int]",50      // Planes 4 and up: Other from U+40000, Control at U+E0001, Other again51      // from U+E0002.52       "  0x2000000,\n  0x7000083,\n  0x7000100,\n]", "const GcbCR : Int = 1", "const GcbRegionalIndicator : Int = 6",53      "const GcbLVT : Int = 13", "const InCBLinker : Int = 3", "const GcbMask : Int = 15",54      "const InCBShift : Int = 4", "const InCBMask : Int = 3", "const ExtendedPictographicBit : Int = 64",55      "fn grapheme_break_value(cp : Int) -> Int",56    ] {57    guard source.contains(expected) else {58      fail("Generated source is missing: \{expected}")59    }60  }61  // The default values need no constants.62  for unexpected in ["const GcbOther", "const InCBNone"] {63    guard !source.contains(unexpected) else {64      fail("Generated source should not contain: \{unexpected}")65    }66  }67}6869///|70test "unknown property values are rejected" {71  try emit_grapheme_break("0041; XX\n", "", "") catch {72    error =>73      inspect(74        error.to_string().contains("Unknown Grapheme_Cluster_Break: XX"),75        content="true",76      )77  } noraise {78    _ => fail("expected an unknown Grapheme_Cluster_Break to be rejected")79  }80  try emit_grapheme_break("", "", "0041; InCB; XX\n") catch {81    error =>82      inspect(83        error.to_string().contains("Unknown Indic_Conjunct_Break: XX"),84        content="true",85      )86  } noraise {87    _ => fail("expected an unknown Indic_Conjunct_Break to be rejected")88  }89}90