unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bidi/bidi_test.mbt
1///|2/// Unit tests for the Unicode Bidirectional Algorithm34///|5test "detect_direction: pure LTR text" {6  inspect(@bidi.detect_direction("Hello World"), content="LTR")7}89///|10test "detect_direction: pure RTL text (Hebrew)" {11  // Hebrew: Shalom12  inspect(13    @bidi.detect_direction("\u{05E9}\u{05DC}\u{05D5}\u{05DD}"),14    content="RTL",15  )16}1718///|19test "detect_direction: pure RTL text (Arabic)" {20  // Arabic: Marhaba (Hello)21  inspect(22    @bidi.detect_direction("\u{0645}\u{0631}\u{062D}\u{0628}\u{0627}"),23    content="RTL",24  )25}2627///|28test "detect_direction: mixed starts with LTR" {29  inspect(30    @bidi.detect_direction("Hello \u{05E9}\u{05DC}\u{05D5}\u{05DD}"),31    content="LTR",32  )33}3435///|36test "detect_direction: mixed starts with RTL" {37  inspect(38    @bidi.detect_direction("\u{05E9}\u{05DC}\u{05D5}\u{05DD} Hello"),39    content="RTL",40  )41}4243///|44test "detect_direction: empty string" {45  inspect(@bidi.detect_direction(""), content="LTR")46}4748///|49test "detect_direction: numbers only" {50  // Numbers are weak types, should default to LTR51  inspect(@bidi.detect_direction("12345"), content="LTR")52}5354///|55test "requires_bidi: pure LTR" {56  inspect(@bidi.requires_bidi("Hello World"), content="false")57}5859///|60test "requires_bidi: contains RTL" {61  inspect(62    @bidi.requires_bidi("Hello \u{05E9}\u{05DC}\u{05D5}\u{05DD}"),63    content="true",64  )65}6667///|68test "requires_bidi: contains Arabic" {69  inspect(@bidi.requires_bidi("\u{0627}\u{0644}\u{0639}"), content="true")70}7172///|73test "process: pure LTR text" {74  let result = @bidi.process("abc")75  inspect(result.base_level, content="0")76  @debug.debug_inspect(result.levels, content="[0, 0, 0]")77}7879///|80test "process: pure RTL text (Hebrew)" {81  // Hebrew letters: alef, bet, gimel82  let result = @bidi.process("\u{05D0}\u{05D1}\u{05D2}")83  inspect(result.base_level, content="1")84  // All RTL characters should be at level 185  @debug.debug_inspect(result.levels, content="[1, 1, 1]")86}8788///|89test "process: mixed LTR and RTL" {90  // "ABC" followed by Hebrew alef, bet, gimel91  let result = @bidi.process("ABC\u{05D0}\u{05D1}\u{05D2}")92  inspect(result.base_level, content="0")93  // LTR at level 0, RTL at level 194  @debug.debug_inspect(result.levels, content="[0, 0, 0, 1, 1, 1]")95}9697///|98test "reorder: pure LTR" {99  let result = @bidi.process("abc")100  let order = @bidi.reorder(result)101  // LTR text should remain in logical order102  @debug.debug_inspect(order, content="[0, 1, 2]")103}104105///|106test "reorder: pure RTL" {107  // Hebrew letters108  let result = @bidi.process("\u{05D0}\u{05D1}\u{05D2}")109  let order = @bidi.reorder(result)110  // RTL text should be reversed111  @debug.debug_inspect(order, content="[2, 1, 0]")112}113114///|115test "reorder: mixed text" {116  // "ab" followed by Hebrew alef, bet117  let result = @bidi.process("ab\u{05D0}\u{05D1}")118  let order = @bidi.reorder(result)119  // LTR stays, RTL reversed120  @debug.debug_inspect(order, content="[0, 1, 3, 2]")121}122123///|124test "reorder_string: pure LTR" {125  let result = @bidi.process("abc")126  inspect(@bidi.reorder_string(result), content="abc")127}128129///|130test "reorder_string: pure RTL" {131  // Hebrew alef, bet, gimel -> gimel, bet, alef visually132  let result = @bidi.process("\u{05D0}\u{05D1}\u{05D2}")133  inspect(@bidi.reorder_string(result), content="\u{05D2}\u{05D1}\u{05D0}")134}135136///|137test "bidi_class: ASCII letters" {138  inspect(@bidi.bidi_class('A'), content="L")139  inspect(@bidi.bidi_class('z'), content="L")140}141142///|143test "bidi_class: Hebrew letters" {144  inspect(@bidi.bidi_class('\u{05D0}'), content="R") // Alef145  inspect(@bidi.bidi_class('\u{05EA}'), content="R") // Tav146}147148///|149test "bidi_class: Arabic letters" {150  inspect(@bidi.bidi_class('\u{0627}'), content="AL") // Alif151  inspect(@bidi.bidi_class('\u{0628}'), content="AL") // Ba152}153154///|155test "bidi_class: unassigned code points use block defaults" {156  inspect(@bidi.bidi_class('\u{0590}'), content="R")157  inspect(@bidi.bidi_class('\u{070E}'), content="AL")158}159160///|161test "bidi_class: supplementary planes" {162  inspect(@bidi.bidi_class('\u{10800}'), content="R")163  inspect(@bidi.bidi_class('\u{1E900}'), content="R")164  inspect(@bidi.bidi_class('\u{1F600}'), content="ON")165  inspect(@bidi.bidi_class('\u{20000}'), content="L")166  // The last code points looked up in the trie, then the packed ranges.167  inspect(@bidi.bidi_class('\u{3FFFD}'), content="L")168  inspect(@bidi.bidi_class('\u{3FFFF}'), content="BN")169  inspect(@bidi.bidi_class('\u{40000}'), content="L")170  inspect(@bidi.bidi_class('\u{E0001}'), content="BN")171  inspect(@bidi.bidi_class('\u{E0100}'), content="NSM")172  inspect(@bidi.bidi_class('\u{E01F0}'), content="BN")173  inspect(@bidi.bidi_class('\u{E1000}'), content="L")174  inspect(@bidi.bidi_class('\u{10FFFD}'), content="L")175  inspect(@bidi.bidi_class('\u{10FFFF}'), content="BN")176}177178///|179test "bidi_class: European numbers" {180  inspect(@bidi.bidi_class('0'), content="EN")181  inspect(@bidi.bidi_class('9'), content="EN")182}183184///|185test "bidi_class: Arabic-Indic digits" {186  inspect(@bidi.bidi_class('\u{0660}'), content="AN") // Arabic-Indic 0187  inspect(@bidi.bidi_class('\u{0669}'), content="AN") // Arabic-Indic 9188}189190///|191test "bidi_class: common separators" {192  inspect(@bidi.bidi_class(','), content="CS")193  inspect(@bidi.bidi_class('.'), content="CS")194}195196///|197test "bidi_class: whitespace" {198  inspect(@bidi.bidi_class(' '), content="WS")199}200201///|202test "get_mirrored: parentheses at RTL level" {203  inspect(@bidi.get_mirrored('(', 1), content=")")204  inspect(@bidi.get_mirrored(')', 1), content="(")205}206207///|208test "get_mirrored: parentheses at LTR level" {209  inspect(@bidi.get_mirrored('(', 0), content="(")210  inspect(@bidi.get_mirrored(')', 0), content=")")211}212213///|214test "get_mirrored: angle brackets" {215  inspect(@bidi.get_mirrored('<', 1), content=">")216  inspect(@bidi.get_mirrored('>', 1), content="<")217}218219///|220test "Direction::level" {221  inspect(@bidi.Direction::LTR.level(), content="0")222  inspect(@bidi.Direction::RTL.level(), content="1")223}224225///|226test "direction_from_level: even levels" {227  inspect(@bidi.direction_from_level(0), content="LTR")228  inspect(@bidi.direction_from_level(2), content="LTR")229  inspect(@bidi.direction_from_level(4), content="LTR")230}231232///|233test "direction_from_level: odd levels" {234  inspect(@bidi.direction_from_level(1), content="RTL")235  inspect(@bidi.direction_from_level(3), content="RTL")236  inspect(@bidi.direction_from_level(5), content="RTL")237}238239///|240test "process: numbers in LTR context" {241  let result = @bidi.process("abc123def")242  // All should be at level 0 (EN promoted to L at LTR level)243  @debug.debug_inspect(result.levels, content="[0, 0, 0, 0, 0, 0, 0, 0, 0]")244}245246///|247test "process: numbers in RTL context" {248  // Hebrew text with numbers249  let result = @bidi.process("\u{05D0}123\u{05D1}")250  inspect(result.base_level, content="1")251  // Hebrew at level 1, numbers at level 2 (EN in RTL context)252  @debug.debug_inspect(result.levels, content="[1, 2, 2, 2, 1]")253}254255///|256test "process: force LTR" {257  // Hebrew text forced to LTR direction258  let result = @bidi.process(259    "\u{05D0}\u{05D1}\u{05D2}",260    direction=@bidi.Direction::LTR,261  )262  inspect(result.base_level, content="0")263}264265///|266test "process: force RTL" {267  // English text forced to RTL direction268  let result = @bidi.process("ABC", direction=@bidi.Direction::RTL)269  inspect(result.base_level, content="1")270}271272///|273/// Resolved classes of a paragraph, as text.274fn resolved(text : String, direction : @bidi.Direction) -> String {275  let classes = @bidi.process(text, direction~).resolved_classes276  classes.map(bc => bc.to_string()).join(" ")277}278279///|280test "weak types: European terminators around European numbers" {281  // W5: terminators before and after a number join it; W7 then makes the282  // whole run L after an L.283  inspect(resolved("a$$1$", LTR), content="L L L L L")284  // After an R the run stays EN.285  inspect(resolved("\u{05D0}$$1$", RTL), content="R EN EN EN EN")286  // W6: terminators that touch no number become ON, which N1 resolves.287  inspect(resolved("\u{05D0}$$\u{05D1}", RTL), content="R R R R")288}289290///|291test "weak types: separators between numbers" {292  // W4: one separator between two numbers of the same kind joins them.293  inspect(resolved("\u{05D0}1,2+3", RTL), content="R EN EN EN EN EN")294  // Two separators in a row do not.295  inspect(resolved("\u{05D0}1,,2", RTL), content="R EN R R EN")296  // W2 and W4: after AL the digits are AN, and only a CS joins two ANs.297  inspect(resolved("\u{0627}1,2+3", RTL), content="R AN AN AN R AN")298}299300///|301test "weak types: NSM takes the type before it" {302  inspect(resolved("\u{0627}\u{0300}1\u{0300}", RTL), content="R R AN AN")303  // After an isolate initiator it becomes ON, which N1 then resolves.304  inspect(resolved("a\u{2066}\u{0300}b\u{2069}", LTR), content="L L L L L")305}306307///|308test "bracket pairs: strong types of nested pairs count for the outer pair" {309  // The only strong type inside the outer pair is inside the inner one.310  inspect(resolved("\u{05D0}([a])", RTL), content="R R R L R R")311  inspect(resolved("a([\u{05D0}])\u{05D1}", LTR), content="L L L R L L R")312  // An unmatched opening bracket inside a pair is ignored.313  inspect(resolved("\u{05D0}(a[)", RTL), content="R R L R R")314}315316///|317test "bracket pairs: marks after a resolved bracket follow it" {318  inspect(319    resolved("\u{05D0}(a)\u{0300}\u{0300}b", RTL),320    content="R R L R R R L",321  )322  inspect(resolved("\u{05D0}(\u{0300}a)", RTL), content="R R R L R")323}324325///|326test "explicit formatting: a lone PDF or PDI still takes the general path" {327  inspect(resolved("a\u{202C}\u{05D0}", LTR), content="L BN R")328  inspect(resolved("a\u{2069}\u{05D0}", LTR), content="L L R")329  @debug.debug_inspect(330    @bidi.process("a\u{202C}\u{05D0}", direction=LTR).levels,331    content="[0, 0, 1]",332  )333}334335///|336test "requires_bidi: finds a character at every offset of a block" {337  for338    marker in [339      "\u{05D0}", "\u{0627}", "\u{202B}", "\u{2067}", "\u{10800}", "\u{1E900}",340    ] {341    for before in 0..<48 {342      for after in [0, 1, 7, 8, 9] {343        let prefix = "a".repeat(before)344        let suffix = "z".repeat(after)345        assert_true(@bidi.requires_bidi(prefix + marker + suffix))346        // Characters above the block limit that need no bidi processing.347        assert_false(@bidi.requires_bidi(prefix + "世😀" + suffix))348      }349    }350  }351}352353///|354test "requires_bidi: no character below the Hebrew block needs it" {355  for cp in 0..<0x590 {356    let c = cp.unsafe_to_char()357    let class = @bidi.bidi_class(c)358    guard !class.is_strong_rtl() && !class.is_explicit() else {359      fail("code point \{cp} requires bidi processing")360    }361  }362  assert_true(@bidi.bidi_class('\u{0590}').is_strong_rtl())363}364