unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

ucd/punctuation/punctuation.mbt
1///|2/// Punctuation (General_Category group P) lookup data3/// Generated from UnicodeData.txt45///|6/// Punctuation trie top level: `cp >> 10` to mid block index7let punctuation_trie_top : Bytes = [8  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,9  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 15,10  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,11  16, 17, 18, 19, 20, 21, 22, 23, 24, 13, 25, 13, 26, 13, 13, 13, 13, 13, 13, 13,12  13, 13, 13, 13, 13, 13, 13, 27, 28, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,13  13, 13, 13, 13, 13, 13, 13, 13, 29, 13, 13, 13, 13, 13, 13, 30, 13, 13, 31, 32,14  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,15  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,16  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,17  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,18  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,19  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,20  13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,21]2223///|24/// Punctuation trie mid level: leaf block indices25let punctuation_trie_mid : Bytes = [26  0, 1, 2, 2, 2, 2, 3, 4, 2, 2, 5, 6, 7, 8, 9, 10, 11, 2, 12, 13, 14, 15, 2, 2, 16,27  17, 2, 18, 19, 2, 20, 21, 22, 23, 2, 2, 2, 2, 24, 2, 25, 2, 2, 2, 26, 27, 28, 29,28  30, 2, 31, 2, 32, 33, 34, 35, 36, 37, 2, 2, 2, 2, 2, 2, 38, 39, 2, 2, 2, 2, 40,29  2, 2, 2, 2, 2, 2, 2, 41, 42, 2, 2, 2, 43, 2, 2, 2, 2, 2, 44, 15, 2, 45, 2, 2, 2,30  46, 47, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 48, 2, 2, 49, 50, 2, 2, 51,31  52, 53, 54, 55, 56, 2, 57, 2, 2, 58, 2, 59, 2, 60, 2, 2, 2, 61, 2, 2, 2, 2, 62,32  2, 2, 63, 2, 2, 2, 2, 2, 64, 2, 65, 2, 66, 67, 68, 69, 2, 2, 26, 2, 2, 70, 71,33  72, 73, 74, 75, 76, 77, 78, 2, 79, 80, 81, 2, 82, 83, 84, 85, 2, 86, 2, 87, 88,34  89, 90, 91, 92, 93, 2, 2, 2, 2, 94, 95, 96, 97, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,35  2, 2, 2, 98, 2, 2, 2, 2, 99, 100, 101, 2, 2, 2, 102, 2, 2, 103, 2, 88, 2, 104,36  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 105, 2, 2, 2, 2, 2, 96, 2, 2, 2, 2, 2, 2, 106,37  2, 2, 2, 2, 2,38]3940///|41/// Punctuation trie leaf level: bitsets of 128 code points, least significant bit first42let punctuation_trie_leaf : Bytes = [43  0, 0, 0, 0, 238, 247, 0, 140, 1, 0, 0, 184, 0, 0, 0, 40, 0, 0, 0, 0, 130, 8, 192,44  136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,45  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,46  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 6, 0, 0, 0,47  0, 0, 64, 73, 0, 0, 0, 0, 0, 24, 0, 0, 54, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60,48  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 255, 63, 0, 0, 0, 0, 0,49  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 3, 0,50  0, 0, 0, 0, 0, 255, 127, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,51  0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0,52  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,53  1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 16, 0, 0, 0, 0, 0, 0, 0,54  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0,55  0, 0, 0, 0, 0, 0, 0, 128, 0, 12, 0, 0, 0, 0, 240, 255, 23, 0, 0, 0, 0, 60, 0, 0,56  0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 6, 0, 0, 0, 0, 0, 0, 0, 0,57  0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,58  0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,59  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0,60  0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0,61  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 7, 0, 0, 0, 0, 255, 7, 0, 0, 0,62  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0,63  0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 63, 0, 0,64  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 252, 1, 0, 0, 224, 0,65  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0,66  0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 8, 0, 0, 0, 0, 0, 0, 0, 255,67  255, 255, 0, 255, 255, 239, 255, 251, 127, 0, 0, 0, 96, 0, 96, 0, 0, 0, 0, 0, 0,68  0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,69  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0,70  0, 192, 255, 0, 0, 248, 255, 255, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 48, 0, 0,71  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 255, 255, 255, 255, 255, 127, 255,72  255, 255, 255, 252, 63, 15, 0, 0, 0, 14, 255, 243, 255, 0, 0, 1, 32, 0, 0, 0, 0,73  0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0,74  0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 64,75  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,76  0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 23, 0, 0, 0,77  0, 0, 192, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 63, 0,78  192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0,79  0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8,80  0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 0, 0, 255,81  255, 255, 255, 247, 255, 11, 13, 0, 0, 238, 247, 0, 140, 1, 0, 0, 184, 0, 0, 0,82  168, 63, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128,83  0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128,84  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0,85  128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 1, 0, 0, 0, 128,86  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0,87  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,88  0, 0, 32, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 3, 0,89  0, 0, 0, 192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,90  0, 128, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 3, 0, 0, 0, 0, 0, 0, 0,91  0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224,92  33, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,93  0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1,94  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,95  0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 255, 255, 0, 0, 0,96  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 255, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0,97  2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0,98  0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0,99  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0,100  0, 128, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,101  0, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,102  0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0,103  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1, 0, 0, 0, 0, 0, 0, 0, 0, 248, 255, 0, 0,104  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0,105  0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0,106  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,107  0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 128, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,108  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 128, 7, 0, 0, 0, 0, 0, 0, 0, 0,109  0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 15, 0, 0, 0,110  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0,111  0,112]113114///|115/// Check whether a character is punctuation (General_Category group P)116pub fn is_punctuation(c : Char) -> Bool {117  let cp = c.to_int()118  if cp >= 0x40000 {119    return false120  }121  let mid = (punctuation_trie_top[cp >> 10].to_int() << 3) | ((cp >> 7) & 7)122  let leaf = punctuation_trie_mid[mid].to_int()123  let bits = punctuation_trie_leaf[(leaf << 4) | ((cp >> 3) & 15)].to_int()124  ((bits >> (cp & 7)) & 1) != 0125}126